2024-04-30T09:19:49.703 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-30T09:19:49.704 DEBUG:teuthology.run:Teuthology command: teuthology --description orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} --archive /home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692 --name adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi --owner scheduled_adking@teuthology --verbose -- /home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692/orig.config.yaml 2024-04-30T09:19:49.750 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-30T09:19:49.841 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692 branch: wip-adk-testing-2024-04-29-2009 description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} email: adking@redhat.com first_in_suite: false job_id: '7680692' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-04-29-2009 ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mgr/cephadm/use_agent: false mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 3a873b0ce83192ee05b56734dad1076a7a94ecc7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 3a873b0ce83192ee05b56734dad1076a7a94ecc7 workunit: branch: wip-adk-testing-2024-04-29-2009 sha1: 3a873b0ce83192ee05b56734dad1076a7a94ecc7 owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b seed: 3042 sha1: 3a873b0ce83192ee05b56734dad1076a7a94ecc7 sleep_before_teardown: 0 subset: 1/16 suite: orch:cephadm suite_branch: wip-adk-testing-2024-04-29-2009 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_3a873b0ce83192ee05b56734dad1076a7a94ecc7/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 3a873b0ce83192ee05b56734dad1076a7a94ecc7 targets: smithi026.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIvrf5XDJXehVvItiF73UHcGXYUQzZpaCBTVhsGK3OpwvamgLxx66itFNXyFn64ATcCOYczjuz15JDFpp/gRvck= smithi046.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD0ZpLm0ZZY5MeL4LntGTEJ8dQjcpYJtJNsqvQqPTl6g+TImH47LGBTkFcYpgXdz1Q1j1P05QqY8izRVkMSuuIM= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 - cephadm.shell: env: - sha1 mon.a: - radosgw-admin realm create --rgw-realm=r --default - radosgw-admin zonegroup create --rgw-zonegroup=default --master --default - radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default - radosgw-admin period update --rgw-realm=r --commit - ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000 - ceph osd pool create foo - rbd pool init foo - ceph orch apply iscsi foo u p - sleep 180 - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch ps - ceph versions - ceph -s - ceph orch ls - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 2' - ceph mgr fail - sleep 180 - ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps --refresh - sleep 180 - ceph orch ps - ceph versions - ceph health detail - ceph -s - ceph mgr fail - sleep 180 - ceph orch ps - ceph versions - ceph -s - ceph health detail - ceph versions | jq -e '.mgr | length == 1' - ceph mgr fail - sleep 180 - ceph orch ps - ceph orch ls - ceph versions - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph versions | jq -e '.mgr | length == 1' - ceph versions | jq -e '.mgr | keys' | grep $sha1 - ceph versions | jq -e '.overall | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 2' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}') - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.mon | length == 1' - ceph versions | jq -e '.mon | keys' | grep $sha1 - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 2' - ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8' - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.osd | length == 1' - ceph versions | jq -e '.osd | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions | jq -e '.rgw | length == 1' - ceph versions | jq -e '.rgw | keys' | grep $sha1 - ceph orch upgrade status - ceph health detail - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 mon.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph orch upgrade status - ceph health detail - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - ceph orch ls | grep '^osd ' - cephadm.shell: mon.a: - ceph orch upgrade ls - ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0 - ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2 teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-30_05:42:49 tube: smithi user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-30T09:19:49.841 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_3a873b0ce83192ee05b56734dad1076a7a94ecc7/qa; will attempt to use it 2024-04-30T09:19:49.842 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_3a873b0ce83192ee05b56734dad1076a7a94ecc7/qa/tasks 2024-04-30T09:19:49.842 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-30T09:19:49.843 INFO:teuthology.task.internal:Checking packages... 2024-04-30T09:19:49.873 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '3a873b0ce83192ee05b56734dad1076a7a94ecc7' 2024-04-30T09:19:49.873 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-30T09:19:49.873 INFO:teuthology.packaging:ref: None 2024-04-30T09:19:49.873 INFO:teuthology.packaging:tag: None 2024-04-30T09:19:49.873 INFO:teuthology.packaging:branch: wip-adk-testing-2024-04-29-2009 2024-04-30T09:19:49.873 INFO:teuthology.packaging:sha1: 3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:19:49.873 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-04-29-2009 2024-04-30T09:19:50.091 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3357.g3a873b0c 2024-04-30T09:19:50.092 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-30T09:19:50.099 INFO:teuthology.task.internal:no buildpackages task found 2024-04-30T09:19:50.099 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-30T09:19:50.105 INFO:teuthology.task.internal:Saving configuration 2024-04-30T09:19:50.123 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-30T09:19:50.129 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-30T09:19:50.152 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-30 09:13:17.215227', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIvrf5XDJXehVvItiF73UHcGXYUQzZpaCBTVhsGK3OpwvamgLxx66itFNXyFn64ATcCOYczjuz15JDFpp/gRvck='} 2024-04-30T09:19:50.176 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi046.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-30 09:13:17.216866', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD0ZpLm0ZZY5MeL4LntGTEJ8dQjcpYJtJNsqvQqPTl6g+TImH47LGBTkFcYpgXdz1Q1j1P05QqY8izRVkMSuuIM='} 2024-04-30T09:19:50.176 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-30T09:19:50.183 INFO:teuthology.task.internal:roles: ubuntu@smithi026.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-30T09:19:50.183 INFO:teuthology.task.internal:roles: ubuntu@smithi046.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-30T09:19:50.183 INFO:teuthology.run_tasks:Running task console_log... 2024-04-30T09:19:50.263 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f2daafe4160>, signals=[15]) 2024-04-30T09:19:50.264 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-30T09:19:50.270 INFO:teuthology.task.internal:Opening connections... 2024-04-30T09:19:50.270 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi026.front.sepia.ceph.com 2024-04-30T09:19:50.272 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-30T09:19:50.349 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi046.front.sepia.ceph.com 2024-04-30T09:19:50.350 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi046.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-30T09:19:50.425 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-30T09:19:50.431 DEBUG:teuthology.orchestra.run.smithi026:> uname -m 2024-04-30T09:19:50.449 INFO:teuthology.orchestra.run.smithi026.stdout:x86_64 2024-04-30T09:19:50.450 DEBUG:teuthology.orchestra.run.smithi026:> cat /etc/os-release 2024-04-30T09:19:50.508 INFO:teuthology.orchestra.run.smithi026.stdout:NAME="CentOS Stream" 2024-04-30T09:19:50.508 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION="9" 2024-04-30T09:19:50.508 INFO:teuthology.orchestra.run.smithi026.stdout:ID="centos" 2024-04-30T09:19:50.508 INFO:teuthology.orchestra.run.smithi026.stdout:ID_LIKE="rhel fedora" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION_ID="9" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:PLATFORM_ID="platform:el9" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:ANSI_COLOR="0;31" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:LOGO="fedora-logo-icon" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:HOME_URL="https://centos.org/" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-30T09:19:50.509 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-30T09:19:50.510 INFO:teuthology.lock.ops:Updating smithi026.front.sepia.ceph.com on lock server 2024-04-30T09:19:50.535 DEBUG:teuthology.orchestra.run.smithi046:> uname -m 2024-04-30T09:19:50.553 INFO:teuthology.orchestra.run.smithi046.stdout:x86_64 2024-04-30T09:19:50.554 DEBUG:teuthology.orchestra.run.smithi046:> cat /etc/os-release 2024-04-30T09:19:50.612 INFO:teuthology.orchestra.run.smithi046.stdout:NAME="CentOS Stream" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:VERSION="9" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:ID="centos" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:ID_LIKE="rhel fedora" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:VERSION_ID="9" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:PLATFORM_ID="platform:el9" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:ANSI_COLOR="0;31" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:LOGO="fedora-logo-icon" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:HOME_URL="https://centos.org/" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-30T09:19:50.613 INFO:teuthology.orchestra.run.smithi046.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-30T09:19:50.614 INFO:teuthology.lock.ops:Updating smithi046.front.sepia.ceph.com on lock server 2024-04-30T09:19:50.635 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-30T09:19:50.644 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-30T09:19:50.649 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-30T09:19:50.650 DEBUG:teuthology.orchestra.run.smithi026:> test '!' -e /home/ubuntu/cephtest 2024-04-30T09:19:50.652 DEBUG:teuthology.orchestra.run.smithi046:> test '!' -e /home/ubuntu/cephtest 2024-04-30T09:19:50.670 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-30T09:19:50.676 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-30T09:19:50.677 DEBUG:teuthology.orchestra.run.smithi026:> test -z $(ls -A /var/lib/ceph) 2024-04-30T09:19:50.710 DEBUG:teuthology.orchestra.run.smithi046:> test -z $(ls -A /var/lib/ceph) 2024-04-30T09:19:50.737 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-30T09:19:50.791 INFO:teuthology.run_tasks:Running task kernel... 2024-04-30T09:19:50.807 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-30T09:19:50.807 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-30T09:19:50.807 DEBUG:teuthology.orchestra.run.smithi026:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-30T09:19:50.808 DEBUG:teuthology.orchestra.run.smithi046:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-30T09:19:50.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-30T09:19:50.826 DEBUG:teuthology.orchestra.run.smithi026:> uname -r 2024-04-30T09:19:50.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-30T09:19:50.827 DEBUG:teuthology.orchestra.run.smithi046:> uname -r 2024-04-30T09:19:50.884 INFO:teuthology.orchestra.run.smithi026.stdout:5.14.0-437.el9.x86_64 2024-04-30T09:19:50.885 INFO:teuthology.task.kernel:Running kernel on smithi026: 5.14.0-437.el9.x86_64 2024-04-30T09:19:50.885 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum install -y kernel 2024-04-30T09:19:50.886 INFO:teuthology.orchestra.run.smithi046.stdout:5.14.0-437.el9.x86_64 2024-04-30T09:19:50.886 INFO:teuthology.task.kernel:Running kernel on smithi046: 5.14.0-437.el9.x86_64 2024-04-30T09:19:50.886 DEBUG:teuthology.orchestra.run.smithi046:> sudo yum install -y kernel 2024-04-30T09:19:53.600 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - BaseOS 7.5 MB/s | 8.0 MB 00:01 2024-04-30T09:19:53.688 INFO:teuthology.orchestra.run.smithi046.stdout:CentOS Stream 9 - BaseOS 7.2 MB/s | 8.0 MB 00:01 2024-04-30T09:19:55.564 INFO:teuthology.orchestra.run.smithi046.stdout:CentOS Stream 9 - AppStream 25 MB/s | 19 MB 00:00 2024-04-30T09:19:55.979 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - AppStream 15 MB/s | 19 MB 00:01 2024-04-30T09:20:00.550 INFO:teuthology.orchestra.run.smithi046.stdout:CentOS Stream 9 - CRB 4.9 MB/s | 6.2 MB 00:01 2024-04-30T09:20:02.249 INFO:teuthology.orchestra.run.smithi046.stdout:CentOS Stream 9 - Extras packages 32 kB/s | 16 kB 00:00 2024-04-30T09:20:03.844 INFO:teuthology.orchestra.run.smithi046.stdout:Extra Packages for Enterprise Linux 14 MB/s | 21 MB 00:01 2024-04-30T09:20:07.377 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - CRB 822 kB/s | 6.2 MB 00:07 2024-04-30T09:20:08.893 INFO:teuthology.orchestra.run.smithi026.stdout:CentOS Stream 9 - Extras packages 52 kB/s | 16 kB 00:00 2024-04-30T09:20:09.050 INFO:teuthology.orchestra.run.smithi046.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-04-30T09:20:10.522 INFO:teuthology.orchestra.run.smithi026.stdout:Extra Packages for Enterprise Linux 14 MB/s | 21 MB 00:01 2024-04-30T09:20:11.015 INFO:teuthology.orchestra.run.smithi046.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-30T09:20:11.015 INFO:teuthology.orchestra.run.smithi046.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-30T09:20:11.104 INFO:teuthology.orchestra.run.smithi046.stdout:Dependencies resolved. 2024-04-30T09:20:11.105 INFO:teuthology.orchestra.run.smithi046.stdout:Nothing to do. 2024-04-30T09:20:11.105 INFO:teuthology.orchestra.run.smithi046.stdout:Complete! 2024-04-30T09:20:11.235 DEBUG:teuthology.orchestra.run.smithi046:> echo no | sudo yum reinstall kernel || true 2024-04-30T09:20:11.760 INFO:teuthology.orchestra.run.smithi046.stdout:Last metadata expiration check: 0:00:02 ago on Tue 30 Apr 2024 09:20:09 AM UTC. 2024-04-30T09:20:11.892 INFO:teuthology.orchestra.run.smithi046.stdout:Dependencies resolved. 2024-04-30T09:20:11.901 INFO:teuthology.orchestra.run.smithi046.stdout:================================================================================ 2024-04-30T09:20:11.901 INFO:teuthology.orchestra.run.smithi046.stdout: Package Architecture Version Repository Size 2024-04-30T09:20:11.901 INFO:teuthology.orchestra.run.smithi046.stdout:================================================================================ 2024-04-30T09:20:11.901 INFO:teuthology.orchestra.run.smithi046.stdout:Reinstalling: 2024-04-30T09:20:11.901 INFO:teuthology.orchestra.run.smithi046.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-30T09:20:11.901 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:20:11.901 INFO:teuthology.orchestra.run.smithi046.stdout:Transaction Summary 2024-04-30T09:20:11.902 INFO:teuthology.orchestra.run.smithi046.stdout:================================================================================ 2024-04-30T09:20:11.902 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:20:11.902 INFO:teuthology.orchestra.run.smithi046.stdout:Total download size: 255 k 2024-04-30T09:20:11.902 INFO:teuthology.orchestra.run.smithi046.stdout:Installed size: 0 2024-04-30T09:20:11.902 INFO:teuthology.orchestra.run.smithi046.stderr:Operation aborted. 2024-04-30T09:20:11.964 INFO:teuthology.orchestra.run.smithi046.stdout:Is this ok [y/N]: 2024-04-30T09:20:11.964 DEBUG:teuthology.orchestra.run.smithi046:> sudo yum reinstall -y kernel || true 2024-04-30T09:20:12.483 INFO:teuthology.orchestra.run.smithi046.stdout:Last metadata expiration check: 0:00:03 ago on Tue 30 Apr 2024 09:20:09 AM UTC. 2024-04-30T09:20:12.604 INFO:teuthology.orchestra.run.smithi046.stdout:Dependencies resolved. 2024-04-30T09:20:12.605 INFO:teuthology.orchestra.run.smithi046.stdout:================================================================================ 2024-04-30T09:20:12.605 INFO:teuthology.orchestra.run.smithi046.stdout: Package Architecture Version Repository Size 2024-04-30T09:20:12.605 INFO:teuthology.orchestra.run.smithi046.stdout:================================================================================ 2024-04-30T09:20:12.605 INFO:teuthology.orchestra.run.smithi046.stdout:Reinstalling: 2024-04-30T09:20:12.605 INFO:teuthology.orchestra.run.smithi046.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-30T09:20:12.605 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:20:12.605 INFO:teuthology.orchestra.run.smithi046.stdout:Transaction Summary 2024-04-30T09:20:12.605 INFO:teuthology.orchestra.run.smithi046.stdout:================================================================================ 2024-04-30T09:20:12.605 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:20:12.606 INFO:teuthology.orchestra.run.smithi046.stdout:Total download size: 255 k 2024-04-30T09:20:12.606 INFO:teuthology.orchestra.run.smithi046.stdout:Installed size: 0 2024-04-30T09:20:12.606 INFO:teuthology.orchestra.run.smithi046.stdout:Downloading Packages: 2024-04-30T09:20:12.986 INFO:teuthology.orchestra.run.smithi046.stdout:kernel-5.14.0-437.el9.x86_64.rpm 2.3 MB/s | 255 kB 00:00 2024-04-30T09:20:12.986 INFO:teuthology.orchestra.run.smithi046.stdout:-------------------------------------------------------------------------------- 2024-04-30T09:20:12.986 INFO:teuthology.orchestra.run.smithi046.stdout:Total 670 kB/s | 255 kB 00:00 2024-04-30T09:20:13.007 INFO:teuthology.orchestra.run.smithi046.stdout:Running transaction check 2024-04-30T09:20:13.198 INFO:teuthology.orchestra.run.smithi046.stdout:Transaction check succeeded. 2024-04-30T09:20:13.198 INFO:teuthology.orchestra.run.smithi046.stdout:Running transaction test 2024-04-30T09:20:13.204 INFO:teuthology.orchestra.run.smithi046.stdout:Transaction test succeeded. 2024-04-30T09:20:13.204 INFO:teuthology.orchestra.run.smithi046.stdout:Running transaction 2024-04-30T09:20:13.334 INFO:teuthology.orchestra.run.smithi046.stdout: Preparing : 1/1 2024-04-30T09:20:13.412 INFO:teuthology.orchestra.run.smithi046.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-30T09:20:13.412 INFO:teuthology.orchestra.run.smithi046.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-30T09:20:14.080 INFO:teuthology.orchestra.run.smithi046.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-30T09:20:14.080 INFO:teuthology.orchestra.run.smithi046.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-30T09:20:14.786 INFO:teuthology.orchestra.run.smithi046.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-30T09:20:14.787 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:20:14.787 INFO:teuthology.orchestra.run.smithi046.stdout:Reinstalled: 2024-04-30T09:20:14.787 INFO:teuthology.orchestra.run.smithi046.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-30T09:20:14.787 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:20:14.787 INFO:teuthology.orchestra.run.smithi046.stdout:Complete! 2024-04-30T09:20:14.985 DEBUG:teuthology.orchestra.run.smithi046:> rpm -q kernel | sort -rV | head -n 1 2024-04-30T09:20:15.034 INFO:teuthology.orchestra.run.smithi046.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-30T09:20:15.034 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-30T09:20:15.034 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-30T09:20:15.034 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-30T09:20:15.035 DEBUG:teuthology.orchestra.run.smithi046:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-30T09:20:15.065 INFO:teuthology.orchestra.run.smithi046.stdout:ttyS1 2024-04-30T09:20:15.084 DEBUG:teuthology.parallel:result is None 2024-04-30T09:20:15.770 INFO:teuthology.orchestra.run.smithi026.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-04-30T09:20:17.735 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-30T09:20:17.735 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-30T09:20:17.831 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-04-30T09:20:17.832 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2024-04-30T09:20:17.833 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2024-04-30T09:20:17.932 DEBUG:teuthology.orchestra.run.smithi026:> echo no | sudo yum reinstall kernel || true 2024-04-30T09:20:18.455 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:03 ago on Tue 30 Apr 2024 09:20:15 AM UTC. 2024-04-30T09:20:18.579 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-04-30T09:20:18.589 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-30T09:20:18.589 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2024-04-30T09:20:18.589 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-30T09:20:18.589 INFO:teuthology.orchestra.run.smithi026.stdout:Reinstalling: 2024-04-30T09:20:18.589 INFO:teuthology.orchestra.run.smithi026.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-30T09:20:18.589 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:20:18.589 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2024-04-30T09:20:18.590 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-30T09:20:18.590 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:20:18.590 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 255 k 2024-04-30T09:20:18.590 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 0 2024-04-30T09:20:18.590 INFO:teuthology.orchestra.run.smithi026.stderr:Operation aborted. 2024-04-30T09:20:18.646 INFO:teuthology.orchestra.run.smithi026.stdout:Is this ok [y/N]: 2024-04-30T09:20:18.646 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum reinstall -y kernel || true 2024-04-30T09:20:19.166 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:04 ago on Tue 30 Apr 2024 09:20:15 AM UTC. 2024-04-30T09:20:19.286 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout:Reinstalling: 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 255 k 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 0 2024-04-30T09:20:19.287 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2024-04-30T09:20:19.438 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-5.14.0-437.el9.x86_64.rpm 2.5 MB/s | 255 kB 00:00 2024-04-30T09:20:19.439 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2024-04-30T09:20:19.439 INFO:teuthology.orchestra.run.smithi026.stdout:Total 1.6 MB/s | 255 kB 00:00 2024-04-30T09:20:19.457 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2024-04-30T09:20:19.656 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2024-04-30T09:20:19.657 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2024-04-30T09:20:19.662 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2024-04-30T09:20:19.662 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2024-04-30T09:20:19.792 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2024-04-30T09:20:19.881 INFO:teuthology.orchestra.run.smithi026.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-30T09:20:19.881 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-30T09:20:20.520 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-30T09:20:20.520 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-30T09:20:21.224 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-30T09:20:21.224 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:20:21.224 INFO:teuthology.orchestra.run.smithi026.stdout:Reinstalled: 2024-04-30T09:20:21.224 INFO:teuthology.orchestra.run.smithi026.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-30T09:20:21.224 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:20:21.224 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2024-04-30T09:20:21.384 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q kernel | sort -rV | head -n 1 2024-04-30T09:20:21.433 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-30T09:20:21.433 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-30T09:20:21.433 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-30T09:20:21.433 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-30T09:20:21.433 DEBUG:teuthology.orchestra.run.smithi026:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-30T09:20:21.468 INFO:teuthology.orchestra.run.smithi026.stdout:ttyS1 2024-04-30T09:20:21.486 DEBUG:teuthology.parallel:result is None 2024-04-30T09:20:21.486 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-30T09:20:21.494 INFO:teuthology.task.internal:Creating test directory... 2024-04-30T09:20:21.494 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-30T09:20:21.529 DEBUG:teuthology.orchestra.run.smithi046:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-30T09:20:21.548 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-30T09:20:21.554 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-30T09:20:21.560 INFO:teuthology.task.internal:Creating archive directory... 2024-04-30T09:20:21.560 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-30T09:20:21.583 DEBUG:teuthology.orchestra.run.smithi046:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-30T09:20:21.623 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-30T09:20:21.632 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-30T09:20:21.632 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-30T09:20:21.646 DEBUG:teuthology.orchestra.run.smithi046:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-30T09:20:21.674 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-30T09:20:21.684 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-30T09:20:21.714 INFO:teuthology.orchestra.run.smithi046.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-30T09:20:21.725 INFO:teuthology.orchestra.run.smithi046.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-30T09:20:21.728 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-30T09:20:21.735 INFO:teuthology.task.internal:Configuring sudo... 2024-04-30T09:20:21.735 DEBUG:teuthology.orchestra.run.smithi026:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-30T09:20:21.737 DEBUG:teuthology.orchestra.run.smithi046:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-30T09:20:21.765 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-30T09:20:21.775 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-30T09:20:21.775 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-30T09:20:21.803 DEBUG:teuthology.orchestra.run.smithi046:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-30T09:20:21.822 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-30T09:20:21.900 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-30T09:20:21.961 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-30T09:20:22.003 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-30T09:20:22.042 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:20:22.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-30T09:20:22.106 DEBUG:teuthology.orchestra.run.smithi046:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-30T09:20:22.150 DEBUG:teuthology.orchestra.run.smithi046:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-30T09:20:22.200 DEBUG:teuthology.orchestra.run.smithi046:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-30T09:20:22.283 DEBUG:teuthology.orchestra.run.smithi046:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-30T09:20:22.328 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-04-30T09:20:22.328 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-30T09:20:22.396 DEBUG:teuthology.orchestra.run.smithi026:> sudo service rsyslog restart 2024-04-30T09:20:22.398 DEBUG:teuthology.orchestra.run.smithi046:> sudo service rsyslog restart 2024-04-30T09:20:22.432 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-30T09:20:22.471 INFO:teuthology.orchestra.run.smithi046.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-30T09:20:22.773 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-30T09:20:22.781 INFO:teuthology.task.internal:Starting timer... 2024-04-30T09:20:22.781 INFO:teuthology.run_tasks:Running task pcp... 2024-04-30T09:20:22.791 INFO:teuthology.run_tasks:Running task selinux... 2024-04-30T09:20:22.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo service auditd rotate 2024-04-30T09:20:22.859 INFO:teuthology.orchestra.run.smithi026.stdout:Rotating logs: 2024-04-30T09:20:22.861 DEBUG:teuthology.orchestra.run.smithi046:> sudo service auditd rotate 2024-04-30T09:20:22.925 INFO:teuthology.orchestra.run.smithi046.stdout:Rotating logs: 2024-04-30T09:20:22.927 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-30T09:20:22.928 DEBUG:teuthology.orchestra.run.smithi026:> /usr/sbin/getenforce 2024-04-30T09:20:22.955 INFO:teuthology.orchestra.run.smithi026.stdout:Permissive 2024-04-30T09:20:22.955 DEBUG:teuthology.orchestra.run.smithi046:> /usr/sbin/getenforce 2024-04-30T09:20:22.981 INFO:teuthology.orchestra.run.smithi046.stdout:Permissive 2024-04-30T09:20:22.981 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi026.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi046.front.sepia.ceph.com': 'permissive'} 2024-04-30T09:20:22.982 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-30T09:20:23.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-30T09:20:23.021 DEBUG:teuthology.orchestra.run.smithi046:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-30T09:20:23.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-30T09:20:23.047 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-30T09:20:23.047 DEBUG:teuthology.orchestra.run.smithi026:> sudo /usr/sbin/setenforce permissive 2024-04-30T09:20:23.086 DEBUG:teuthology.orchestra.run.smithi046:> sudo /usr/sbin/setenforce permissive 2024-04-30T09:20:23.118 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-30T09:20:23.129 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-30T09:20:23.146 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-30T09:20:23.147 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi026.front.sepia.ceph.com,smithi046.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-30T09:24:25.642 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi026.front.sepia.ceph.com'), Remote(name='ubuntu@smithi046.front.sepia.ceph.com')] 2024-04-30T09:24:25.643 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-04-30T09:24:25.644 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-30T09:24:25.731 DEBUG:teuthology.orchestra.run.smithi026:> true 2024-04-30T09:24:25.822 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi026.front.sepia.ceph.com' 2024-04-30T09:24:25.823 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi046.front.sepia.ceph.com' 2024-04-30T09:24:25.823 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi046.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-30T09:24:25.902 DEBUG:teuthology.orchestra.run.smithi046:> true 2024-04-30T09:24:25.985 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi046.front.sepia.ceph.com' 2024-04-30T09:24:25.986 INFO:teuthology.run_tasks:Running task clock... 2024-04-30T09:24:25.997 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-30T09:24:25.997 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-30T09:24:25.997 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-30T09:24:26.000 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-30T09:24:26.000 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-30T09:24:26.028 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-30T09:24:26.042 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-30T09:24:26.054 INFO:teuthology.orchestra.run.smithi046.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-30T09:24:26.067 INFO:teuthology.orchestra.run.smithi026.stderr:sudo: ntpd: command not found 2024-04-30T09:24:26.067 INFO:teuthology.orchestra.run.smithi046.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-30T09:24:26.078 INFO:teuthology.orchestra.run.smithi026.stdout:506 Cannot talk to daemon 2024-04-30T09:24:26.091 INFO:teuthology.orchestra.run.smithi046.stderr:sudo: ntpd: command not found 2024-04-30T09:24:26.092 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-30T09:24:26.102 INFO:teuthology.orchestra.run.smithi046.stdout:506 Cannot talk to daemon 2024-04-30T09:24:26.106 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-30T09:24:26.115 INFO:teuthology.orchestra.run.smithi046.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-30T09:24:26.129 INFO:teuthology.orchestra.run.smithi046.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-30T09:24:26.163 INFO:teuthology.orchestra.run.smithi026.stderr:bash: line 1: ntpq: command not found 2024-04-30T09:24:26.167 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-30T09:24:26.167 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2024-04-30T09:24:26.167 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T09:24:26.167 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T09:24:26.167 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T09:24:26.167 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T09:24:26.184 INFO:teuthology.orchestra.run.smithi046.stderr:bash: line 1: ntpq: command not found 2024-04-30T09:24:26.187 INFO:teuthology.orchestra.run.smithi046.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-30T09:24:26.187 INFO:teuthology.orchestra.run.smithi046.stdout:=============================================================================== 2024-04-30T09:24:26.187 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T09:24:26.187 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T09:24:26.187 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T09:24:26.187 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T09:24:26.188 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-30T09:24:26.283 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1, 'mgr/cephadm/use_agent': False}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '3a873b0ce83192ee05b56734dad1076a7a94ecc7'} 2024-04-30T09:24:26.283 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-30T09:24:26.283 INFO:tasks.cephadm:Cluster fsid is 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:24:26.283 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-30T09:24:26.284 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.26', 'mon.c': '[v2:172.21.15.26:3301,v1:172.21.15.26:6790]', 'mon.b': '172.21.15.46'} 2024-04-30T09:24:26.284 INFO:tasks.cephadm:First mon is mon.a on smithi026 2024-04-30T09:24:26.284 INFO:tasks.cephadm:First mgr is y 2024-04-30T09:24:26.284 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-30T09:24:26.284 DEBUG:teuthology.orchestra.run.smithi026:> sudo hostname $(hostname -s) 2024-04-30T09:24:26.310 DEBUG:teuthology.orchestra.run.smithi046:> sudo hostname $(hostname -s) 2024-04-30T09:24:26.335 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-30T09:24:26.335 DEBUG:teuthology.orchestra.run.smithi026:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-30T09:24:26.607 INFO:teuthology.orchestra.run.smithi026.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 30 09:24 /home/ubuntu/cephtest/cephadm 2024-04-30T09:24:26.608 DEBUG:teuthology.orchestra.run.smithi046:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-30T09:24:26.780 INFO:teuthology.orchestra.run.smithi046.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 30 09:24 /home/ubuntu/cephtest/cephadm 2024-04-30T09:24:26.781 DEBUG:teuthology.orchestra.run.smithi026:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-30T09:24:26.797 DEBUG:teuthology.orchestra.run.smithi046:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-30T09:24:26.823 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-30T09:24:26.823 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-30T09:24:26.840 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-30T09:24:28.343 INFO:teuthology.orchestra.run.smithi026.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-30T09:24:28.407 INFO:teuthology.orchestra.run.smithi046.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-30T09:24:49.789 INFO:teuthology.orchestra.run.smithi046.stdout:{ 2024-04-30T09:24:49.789 INFO:teuthology.orchestra.run.smithi046.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-30T09:24:49.790 INFO:teuthology.orchestra.run.smithi046.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-30T09:24:49.790 INFO:teuthology.orchestra.run.smithi046.stdout: "repo_digests": [ 2024-04-30T09:24:49.790 INFO:teuthology.orchestra.run.smithi046.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-30T09:24:49.790 INFO:teuthology.orchestra.run.smithi046.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-30T09:24:49.790 INFO:teuthology.orchestra.run.smithi046.stdout: ] 2024-04-30T09:24:49.790 INFO:teuthology.orchestra.run.smithi046.stdout:} 2024-04-30T09:24:51.596 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:24:51.596 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-30T09:24:51.596 INFO:teuthology.orchestra.run.smithi026.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-30T09:24:51.596 INFO:teuthology.orchestra.run.smithi026.stdout: "repo_digests": [ 2024-04-30T09:24:51.596 INFO:teuthology.orchestra.run.smithi026.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-30T09:24:51.596 INFO:teuthology.orchestra.run.smithi026.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-30T09:24:51.596 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2024-04-30T09:24:51.596 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:24:51.617 DEBUG:teuthology.orchestra.run.smithi026:> sudo mkdir -p /etc/ceph 2024-04-30T09:24:51.676 DEBUG:teuthology.orchestra.run.smithi046:> sudo mkdir -p /etc/ceph 2024-04-30T09:24:51.709 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 777 /etc/ceph 2024-04-30T09:24:51.735 DEBUG:teuthology.orchestra.run.smithi046:> sudo chmod 777 /etc/ceph 2024-04-30T09:24:51.779 INFO:tasks.cephadm:Writing seed config... 2024-04-30T09:24:51.780 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-04-30T09:24:51.780 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-30T09:24:51.780 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-30T09:24:51.780 INFO:tasks.cephadm: override: [mgr] mgr/cephadm/use_agent = False 2024-04-30T09:24:51.780 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-30T09:24:51.780 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-30T09:24:51.780 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-30T09:24:51.780 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-30T09:24:51.780 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-30T09:24:51.781 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:24:51.781 DEBUG:teuthology.orchestra.run.smithi026:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-30T09:24:51.797 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 703f4aea-06d3-11ef-bc93-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 mgr/cephadm/use_agent = False [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-30T09:24:51.797 DEBUG:teuthology.orchestra.run.smithi026:mon.a> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.a.service 2024-04-30T09:24:51.840 DEBUG:teuthology.orchestra.run.smithi026:mgr.y> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.y.service 2024-04-30T09:24:51.882 INFO:tasks.cephadm:Bootstrapping... 2024-04-30T09:24:51.882 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.26 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-30T09:24:52.083 INFO:teuthology.orchestra.run.smithi026.stderr:-------------------------------------------------------------------------------- 2024-04-30T09:24:52.083 INFO:teuthology.orchestra.run.smithi026.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '703f4aea-06d3-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.26', '--skip-admin-label'] 2024-04-30T09:24:52.107 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: 4.9.4-dev 2024-04-30T09:24:52.109 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying podman|docker is present... 2024-04-30T09:24:52.131 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: 4.9.4-dev 2024-04-30T09:24:52.133 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying lvm2 is present... 2024-04-30T09:24:52.134 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying time synchronization is in place... 2024-04-30T09:24:52.140 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-30T09:24:52.147 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-30T09:24:52.153 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: enabled 2024-04-30T09:24:52.159 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: active 2024-04-30T09:24:52.160 INFO:teuthology.orchestra.run.smithi026.stderr:Unit chronyd.service is enabled and running 2024-04-30T09:24:52.160 INFO:teuthology.orchestra.run.smithi026.stderr:Repeating the final host check... 2024-04-30T09:24:52.182 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: 4.9.4-dev 2024-04-30T09:24:52.185 INFO:teuthology.orchestra.run.smithi026.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-30T09:24:52.185 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl is present 2024-04-30T09:24:52.185 INFO:teuthology.orchestra.run.smithi026.stderr:lvcreate is present 2024-04-30T09:24:52.191 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-30T09:24:52.197 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-30T09:24:52.204 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: enabled 2024-04-30T09:24:52.210 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: active 2024-04-30T09:24:52.211 INFO:teuthology.orchestra.run.smithi026.stderr:Unit chronyd.service is enabled and running 2024-04-30T09:24:52.211 INFO:teuthology.orchestra.run.smithi026.stderr:Host looks OK 2024-04-30T09:24:52.211 INFO:teuthology.orchestra.run.smithi026.stderr:Cluster fsid: 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:24:52.211 INFO:teuthology.orchestra.run.smithi026.stderr:Acquiring lock 140442326921072 on /run/cephadm/703f4aea-06d3-11ef-bc93-c7b262605968.lock 2024-04-30T09:24:52.211 INFO:teuthology.orchestra.run.smithi026.stderr:Lock 140442326921072 acquired on /run/cephadm/703f4aea-06d3-11ef-bc93-c7b262605968.lock 2024-04-30T09:24:52.212 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying IP 172.21.15.26 port 3300 ... 2024-04-30T09:24:52.212 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying IP 172.21.15.26 port 6789 ... 2024-04-30T09:24:52.212 INFO:teuthology.orchestra.run.smithi026.stderr:Base mon IP is 172.21.15.26, final addrv is [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2024-04-30T09:24:52.214 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.26 metric 100 2024-04-30T09:24:52.214 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.26 metric 100 2024-04-30T09:24:52.217 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-30T09:24:52.217 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 1024 pref medium 2024-04-30T09:24:52.217 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 100 pref medium 2024-04-30T09:24:52.219 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-30T09:24:52.219 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-30T09:24:52.219 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-30T09:24:52.219 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: 4: ens1f0: mtu 1500 state UP qlen 1000 2024-04-30T09:24:52.220 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: inet6 fe80::ae1f:6bff:fea5:cac0/64 scope link noprefixroute 2024-04-30T09:24:52.220 INFO:teuthology.orchestra.run.smithi026.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-30T09:24:52.220 INFO:teuthology.orchestra.run.smithi026.stderr:Mon IP `172.21.15.26` is in CIDR network `172.21.0.0/20` 2024-04-30T09:24:52.220 INFO:teuthology.orchestra.run.smithi026.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-30T09:24:52.221 INFO:teuthology.orchestra.run.smithi026.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-30T09:24:52.327 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-30T09:24:52.704 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Getting image source signatures 2024-04-30T09:24:52.704 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-30T09:24:52.704 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-30T09:24:52.704 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-30T09:24:52.704 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-30T09:24:52.704 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-30T09:24:52.704 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-30T09:24:52.863 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: Writing manifest to image destination 2024-04-30T09:24:53.016 INFO:teuthology.orchestra.run.smithi026.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-30T09:24:53.432 INFO:teuthology.orchestra.run.smithi026.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-30T09:24:53.949 INFO:teuthology.orchestra.run.smithi026.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-30T09:24:53.949 INFO:teuthology.orchestra.run.smithi026.stderr:Extracting ceph user uid/gid from container image... 2024-04-30T09:24:54.349 INFO:teuthology.orchestra.run.smithi026.stderr:stat: 167 167 2024-04-30T09:24:54.880 INFO:teuthology.orchestra.run.smithi026.stderr:Creating initial keys... 2024-04-30T09:24:55.454 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph-authtool: AQDnuDBmF08bFxAAxx+9a2am7pcPgulmxiyv0w== 2024-04-30T09:24:56.418 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph-authtool: AQDouDBmF8LjExAAQ6uGc6ZGUiJHtpsP9OEIqg== 2024-04-30T09:24:57.356 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph-authtool: AQDpuDBmuh3EDxAADMnTyQDETHCk83Gn1ACoqA== 2024-04-30T09:24:57.816 INFO:teuthology.orchestra.run.smithi026.stderr:Creating initial monmap... 2024-04-30T09:24:58.196 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-30T09:24:58.196 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-30T09:24:58.196 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:24:58.196 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-30T09:24:58.686 INFO:teuthology.orchestra.run.smithi026.stderr:monmaptool for a [v2:172.21.15.26:3300,v1:172.21.15.26:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-30T09:24:58.686 INFO:teuthology.orchestra.run.smithi026.stderr:setting min_mon_release = octopus 2024-04-30T09:24:58.686 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: set fsid to 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:24:58.687 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-30T09:24:58.687 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T09:24:58.687 INFO:teuthology.orchestra.run.smithi026.stderr:Creating mon... 2024-04-30T09:24:59.592 INFO:teuthology.orchestra.run.smithi026.stderr:create mon.a on 2024-04-30T09:24:59.803 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-30T09:24:59.956 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-703f4aea-06d3-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-703f4aea-06d3-11ef-bc93-c7b262605968.target. 2024-04-30T09:24:59.956 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-703f4aea-06d3-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-703f4aea-06d3-11ef-bc93-c7b262605968.target. 2024-04-30T09:25:00.254 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Failed to reset failed state of unit ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.a.service: Unit ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.a.service not loaded. 2024-04-30T09:25:00.262 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/ceph-703f4aea-06d3-11ef-bc93-c7b262605968.target.wants/ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.a.service → /etc/systemd/system/ceph-703f4aea-06d3-11ef-bc93-c7b262605968@.service. 2024-04-30T09:25:00.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:00 smithi026 ceph-mon[22002]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-30T09:25:00.966 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: disabled 2024-04-30T09:25:00.973 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-30T09:25:00.974 INFO:teuthology.orchestra.run.smithi026.stderr:firewalld.service is not enabled 2024-04-30T09:25:00.974 INFO:teuthology.orchestra.run.smithi026.stderr:Not possible to enable service . firewalld.service is not available 2024-04-30T09:25:00.974 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mon to start... 2024-04-30T09:25:00.975 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mon... 2024-04-30T09:25:01.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:00 smithi026 bash[21984]: 53fa8c505465967ded567880673a03584ed848b90e4a1009eb1553dae955013a 2024-04-30T09:25:01.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:00 smithi026 systemd[1]: Started Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:25:01.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: cluster: 2024-04-30T09:25:01.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: id: 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:25:01.506 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: services: 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon: 1 daemons, quorum a (age 0.604008s) 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: data: 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: pgs: 2024-04-30T09:25:01.507 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:02.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:01 smithi026 ceph-mon[22002]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-30T09:25:02.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:01 smithi026 ceph-mon[22002]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} 2024-04-30T09:25:02.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:01 smithi026 ceph-mon[22002]: fsmap 2024-04-30T09:25:02.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:01 smithi026 ceph-mon[22002]: osdmap e1: 0 total, 0 up, 0 in 2024-04-30T09:25:02.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:01 smithi026 ceph-mon[22002]: mgrmap e1: no daemons active 2024-04-30T09:25:02.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:01 smithi026 ceph-mon[22002]: from='client.? 172.21.15.26:0/3425466636' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:25:02.096 INFO:teuthology.orchestra.run.smithi026.stderr:mon is available 2024-04-30T09:25:02.096 INFO:teuthology.orchestra.run.smithi026.stderr:Assimilating anything we can from ceph.conf... 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [global] 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: fsid = 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [mgr] 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mgr/cephadm/use_agent = False 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [osd] 2024-04-30T09:25:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-30T09:25:02.604 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-30T09:25:03.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:02 smithi026 ceph-mon[22002]: from='client.? 172.21.15.26:0/1301948374' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-30T09:25:03.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:02 smithi026 ceph-mon[22002]: from='client.? 172.21.15.26:0/1301948374' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-30T09:25:03.236 INFO:teuthology.orchestra.run.smithi026.stderr:Generating new minimal ceph.conf... 2024-04-30T09:25:04.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:03 smithi026 ceph-mon[22002]: from='client.? 172.21.15.26:0/2769423455' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:25:04.192 INFO:teuthology.orchestra.run.smithi026.stderr:Restarting the monitor... 2024-04-30T09:25:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:04 smithi026 systemd[1]: Stopping Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:25:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:04 smithi026 bash[22232]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-30T09:25:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a[21998]: 2024-04-30T09:25:04.349+0000 7f6861c05700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T09:25:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a[21998]: 2024-04-30T09:25:04.349+0000 7f6861c05700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-30T09:25:04.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:04 smithi026 podman[22245]: 2024-04-30 09:25:04.514181179 +0000 UTC m=+0.268756267 container died 53fa8c505465967ded567880673a03584ed848b90e4a1009eb1553dae955013a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, com.redhat.component=centos-stream-container, architecture=x86_64, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., vcs-type=git, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.openshift.expose-services=, version=8, io.buildah.version=1.19.8, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-30T09:25:04.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:04 smithi026 podman[22245]: 2024-04-30 09:25:04.799972457 +0000 UTC m=+0.554547545 container cleanup 53fa8c505465967ded567880673a03584ed848b90e4a1009eb1553dae955013a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, architecture=x86_64, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, name=centos-stream, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_CLEAN=True, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, io.buildah.version=1.19.8, version=8, RELEASE=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git) 2024-04-30T09:25:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:04 smithi026 bash[22245]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a 2024-04-30T09:25:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:04 smithi026 bash[22271]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon.a" found: no such container 2024-04-30T09:25:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 podman[22259]: 2024-04-30 09:25:05.009130046 +0000 UTC m=+0.494323142 container remove 53fa8c505465967ded567880673a03584ed848b90e4a1009eb1553dae955013a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, RELEASE=HEAD, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, version=8, com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.expose-services=, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, release=754) 2024-04-30T09:25:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-30T09:25:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 systemd[1]: Stopped Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:25:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 systemd[1]: Starting Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:25:05.529 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 podman[22343]: 2024-04-30 09:25:05.243700088 +0000 UTC m=+0.112253842 container create b6420404af4cdb8d1fe0e2b9186e5a1d95b7549bfc023e1fb5ce49531f43c725 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, maintainer=Guillaume Abrioux , ceph=True, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, release=754, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, name=centos-stream, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.openshift.expose-services=, vcs-type=git, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD) 2024-04-30T09:25:05.529 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 podman[22343]: 2024-04-30 09:25:05.150619051 +0000 UTC m=+0.019172778 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 podman[22343]: 2024-04-30 09:25:05.420647081 +0000 UTC m=+0.289200793 container init b6420404af4cdb8d1fe0e2b9186e5a1d95b7549bfc023e1fb5ce49531f43c725 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, RELEASE=HEAD, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, io.buildah.version=1.19.8, GIT_CLEAN=True, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, vendor=Red Hat, Inc.) 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 podman[22343]: 2024-04-30 09:25:05.423492939 +0000 UTC m=+0.292046652 container start b6420404af4cdb8d1fe0e2b9186e5a1d95b7549bfc023e1fb5ce49531f43c725 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, release=754, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, RELEASE=HEAD, GIT_CLEAN=True, architecture=x86_64, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.display-name=CentOS Stream 8) 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: set uid:gid to 167:167 (ceph:ceph) 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: pidfile_write: ignore empty --pid-file 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: load: jerasure load: lrc 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: RocksDB version: 6.15.5 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Compile date Apr 18 2022 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: DB SUMMARY 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: DB Session ID: KWC2SQIUCIXT2YAMLNEY 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: CURRENT file: CURRENT 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: IDENTITY file: IDENTITY 2024-04-30T09:25:05.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 82635 ; 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.error_if_exists: 0 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.create_if_missing: 0 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.paranoid_checks: 1 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.env: 0x561d58ac6860 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.fs: Posix File System 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.info_log: 0x561d5a9dfdc0 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_file_opening_threads: 16 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.statistics: (nil) 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.use_fsync: 0 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_log_file_size: 0 2024-04-30T09:25:05.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.keep_log_file_num: 1000 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.recycle_log_file_num: 0 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.allow_fallocate: 1 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.allow_mmap_reads: 0 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.allow_mmap_writes: 0 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.use_direct_reads: 0 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.create_missing_column_families: 0 2024-04-30T09:25:05.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.db_log_dir: 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.advise_random_on_open: 1 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.db_write_buffer_size: 0 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.write_buffer_manager: 0x561d5aad02a0 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-30T09:25:05.534 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.rate_limiter: (nil) 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.wal_recovery_mode: 2 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.enable_thread_tracking: 0 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.enable_pipelined_write: 0 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.unordered_write: 0 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.row_cache: None 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.wal_filter: None 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-30T09:25:05.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.allow_ingest_behind: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.preserve_deletes: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.two_write_queues: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.manual_wal_flush: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.atomic_flush: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.log_readahead_size: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.best_efforts_recovery: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.allow_data_in_errors: 0 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.db_host_id: __hostname__ 2024-04-30T09:25:05.536 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_background_jobs: 2 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_background_compactions: -1 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_subcompactions: 1 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_total_wal_size: 0 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_open_files: -1 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bytes_per_sync: 0 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-30T09:25:05.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_readahead_size: 0 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_background_flushes: -1 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Compression algorithms supported: 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: kZSTD supported: 0 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: kXpressCompression supported: 0 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: kLZ4HCCompression supported: 1 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: kLZ4Compression supported: 1 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: kBZip2Compression supported: 0 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: kZlibCompression supported: 1 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: kSnappyCompression supported: 1 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-30T09:25:05.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.merge_operator: 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_filter: None 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_filter_factory: None 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.sst_partitioner_factory: None 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561d5a9abd00) 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-30T09:25:05.539 INFO:journalctl@ceph.mon.a.smithi026.stdout: hash_index_allow_collision: 1 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 1 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x561d5aa16170 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2024-04-30T09:25:05.540 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 4 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.write_buffer_size: 33554432 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_write_buffer_number: 2 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compression: NoCompression 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bottommost_compression: Disabled 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.prefix_extractor: nullptr 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-30T09:25:05.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.num_levels: 7 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compression_opts.level: 32767 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compression_opts.strategy: 0 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-30T09:25:05.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compression_opts.enabled: false 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.target_file_size_base: 67108864 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-30T09:25:05.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.arena_block_size: 4194304 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.disable_auto_compactions: 0 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-30T09:25:05.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.table_properties_collectors: 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.inplace_update_support: 0 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.bloom_locality: 0 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.max_successive_merges: 0 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.paranoid_file_checks: 0 2024-04-30T09:25:05.545 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.force_consistency_checks: 1 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.report_bg_io_stats: 0 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.ttl: 2592000 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.enable_blob_files: false 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.min_blob_size: 0 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.blob_file_size: 268435456 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714469105450071, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-30T09:25:05.546 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714469105450823, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79185, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77448, "index_size": 225, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10027, "raw_average_key_size": 47, "raw_value_size": 71757, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714469105, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "94a0d9a8-7240-4157-afeb-38bd9e2fe26a", "db_session_id": "KWC2SQIUCIXT2YAMLNEY"}} 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714469105451162, "job": 1, "event": "recovery_finished"} 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x561d5a9fca80 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: DB pointer 0x561d5aa0c000 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-30T09:25:05.547 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:25:05.548 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 24.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 24.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 79.09 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-04-30T09:25:05.549 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 79.09 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 155.7 0.00 0.00 1 0.000 0 0 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 24.54 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:25:05.550 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: mon.a@-1(???) e1 preinit fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: mon.a@-1(???).mds e1 new map 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: mon.a@-1(???).mds e1 print_map 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout: e1 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: -1 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout: No filesystems configured 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-30T09:25:05.551 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-30T09:25:05.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-30T09:25:05.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-30T09:25:05.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 ceph-mon[22362]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-30T09:25:05.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 bash[22343]: b6420404af4cdb8d1fe0e2b9186e5a1d95b7549bfc023e1fb5ce49531f43c725 2024-04-30T09:25:05.552 INFO:teuthology.orchestra.run.smithi026.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-30T09:25:05.784 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:05 smithi026 systemd[1]: Started Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:25:06.547 INFO:teuthology.orchestra.run.smithi026.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-30T09:25:06.548 INFO:teuthology.orchestra.run.smithi026.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-30T09:25:06.548 INFO:teuthology.orchestra.run.smithi026.stderr:Creating mgr... 2024-04-30T09:25:06.548 INFO:teuthology.orchestra.run.smithi026.stderr:Verifying port 9283 ... 2024-04-30T09:25:06.727 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Failed to reset failed state of unit ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.y.service: Unit ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.y.service not loaded. 2024-04-30T09:25:06.735 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: Created symlink /etc/systemd/system/ceph-703f4aea-06d3-11ef-bc93-c7b262605968.target.wants/ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-703f4aea-06d3-11ef-bc93-c7b262605968@.service. 2024-04-30T09:25:06.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:06 smithi026 ceph-mon[22362]: mon.a is new leader, mons a in quorum (ranks 0) 2024-04-30T09:25:06.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:06 smithi026 ceph-mon[22362]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} 2024-04-30T09:25:06.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:06 smithi026 ceph-mon[22362]: fsmap 2024-04-30T09:25:06.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:06 smithi026 ceph-mon[22362]: osdmap e1: 0 total, 0 up, 0 in 2024-04-30T09:25:06.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:06 smithi026 ceph-mon[22362]: mgrmap e1: no daemons active 2024-04-30T09:25:06.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:06 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3682185669' entity='client.admin' 2024-04-30T09:25:07.104 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:06 smithi026 systemd[1]: Starting Ceph mgr.y for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:25:07.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:07 smithi026 podman[22576]: 2024-04-30 09:25:07.103493578 +0000 UTC m=+0.107022968 container create 780f8f988439fad6b67ea8ab1c9475d152b52e0f13596cd69eb996f6c90c5064 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.expose-services=, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, name=centos-stream, RELEASE=HEAD, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-30T09:25:07.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:07 smithi026 podman[22576]: 2024-04-30 09:25:07.014606819 +0000 UTC m=+0.018136180 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-30T09:25:07.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:07 smithi026 podman[22576]: 2024-04-30 09:25:07.280628006 +0000 UTC m=+0.284157355 container init 780f8f988439fad6b67ea8ab1c9475d152b52e0f13596cd69eb996f6c90c5064 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, release=754, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-30T09:25:07.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:07 smithi026 podman[22576]: 2024-04-30 09:25:07.283326344 +0000 UTC m=+0.286855693 container start 780f8f988439fad6b67ea8ab1c9475d152b52e0f13596cd69eb996f6c90c5064 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, com.redhat.component=centos-stream-container, name=centos-stream, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, release=754, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0) 2024-04-30T09:25:07.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:07 smithi026 bash[22576]: 780f8f988439fad6b67ea8ab1c9475d152b52e0f13596cd69eb996f6c90c5064 2024-04-30T09:25:07.376 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: disabled 2024-04-30T09:25:07.384 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-30T09:25:07.385 INFO:teuthology.orchestra.run.smithi026.stderr:firewalld.service is not enabled 2024-04-30T09:25:07.385 INFO:teuthology.orchestra.run.smithi026.stderr:Not possible to enable service . firewalld.service is not available 2024-04-30T09:25:07.393 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: disabled 2024-04-30T09:25:07.400 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-30T09:25:07.402 INFO:teuthology.orchestra.run.smithi026.stderr:firewalld.service is not enabled 2024-04-30T09:25:07.402 INFO:teuthology.orchestra.run.smithi026.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-30T09:25:07.403 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mgr to start... 2024-04-30T09:25:07.403 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mgr... 2024-04-30T09:25:07.651 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:07 smithi026 systemd[1]: Started Ceph mgr.y for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:25:07.651 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:07 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:07.434+0000 7f312152a000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-30T09:25:07.651 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:07 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:07.569+0000 7f312152a000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:25:07.923 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:07 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:07.672+0000 7f312152a000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-30T09:25:07.925 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:07.925 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-30T09:25:07.925 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsid": "703f4aea-06d3-11ef-bc93-c7b262605968", 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "health": { 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "checks": {}, 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mutes": [] 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum": [ 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 0 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "a" 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:07.926 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "monmap": { 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osdmap": { 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-30T09:25:07.927 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-30T09:25:07.930 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-30T09:25:07.930 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgmap": { 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:07.931 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsmap": { 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": false, 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modules": [ 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "iostat", 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "nfs", 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "restful" 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "servicemap": { 2024-04-30T09:25:07.932 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:07.933 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modified": "2024-04-30T09:25:00.900690+0000", 2024-04-30T09:25:07.933 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-30T09:25:07.933 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:07.933 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-30T09:25:07.933 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-30T09:25:08.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:07 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1894538434' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-30T09:25:08.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:08 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:08.004+0000 7f312152a000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-30T09:25:08.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:08 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:08.089+0000 7f312152a000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-30T09:25:08.448 INFO:teuthology.orchestra.run.smithi026.stderr:mgr not available, waiting (1/15)... 2024-04-30T09:25:09.554 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:09 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:09.298+0000 7f312152a000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-30T09:25:09.554 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:09 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:09.466+0000 7f312152a000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-30T09:25:09.554 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:09 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:09.550+0000 7f312152a000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-30T09:25:10.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:09 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:09.872+0000 7f312152a000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-30T09:25:10.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:09 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:09.964+0000 7f312152a000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-30T09:25:10.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:10 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:10.059+0000 7f312152a000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-30T09:25:10.574 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:10 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:10.308+0000 7f312152a000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:25:10.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:10 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:10.772+0000 7f312152a000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-30T09:25:10.982 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:10.982 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsid": "703f4aea-06d3-11ef-bc93-c7b262605968", 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "health": { 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "checks": {}, 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mutes": [] 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum": [ 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 0 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "a" 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "monmap": { 2024-04-30T09:25:10.983 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osdmap": { 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgmap": { 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-30T09:25:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsmap": { 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": false, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-30T09:25:10.985 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modules": [ 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "iostat", 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "nfs", 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "restful" 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "servicemap": { 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modified": "2024-04-30T09:25:00.900690+0000", 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-30T09:25:10.986 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-30T09:25:11.487 INFO:teuthology.orchestra.run.smithi026.stderr:mgr not available, waiting (2/15)... 2024-04-30T09:25:11.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:11 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1970849699' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-30T09:25:11.617 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:11 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:11.289+0000 7f312152a000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-30T09:25:11.617 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:11 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:11.385+0000 7f312152a000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-30T09:25:11.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:11 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:11.691+0000 7f312152a000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-30T09:25:11.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:11 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:11.782+0000 7f312152a000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-30T09:25:12.575 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:12.213+0000 7f312152a000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-30T09:25:12.575 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:12.349+0000 7f312152a000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-30T09:25:12.575 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:12.456+0000 7f312152a000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-30T09:25:12.575 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:12.574+0000 7f312152a000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-30T09:25:12.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:12.665+0000 7f312152a000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-30T09:25:13.594 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:13.233+0000 7f312152a000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-30T09:25:13.896 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:13.616+0000 7f312152a000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-30T09:25:13.896 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: Activating manager daemon y 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: mgrmap e2: y(active, starting, since 0.00261213s) 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: Manager daemon y is now available 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' 2024-04-30T09:25:13.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:13 smithi026 ceph-mon[22362]: from='mgr.14100 172.21.15.26:0/3401194832' entity='mgr.y' 2024-04-30T09:25:13.993 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:13.993 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-30T09:25:13.993 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsid": "703f4aea-06d3-11ef-bc93-c7b262605968", 2024-04-30T09:25:13.993 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "health": { 2024-04-30T09:25:13.993 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-30T09:25:13.993 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "checks": {}, 2024-04-30T09:25:13.993 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mutes": [] 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum": [ 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 0 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "a" 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_age": 8, 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "monmap": { 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-30T09:25:13.994 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osdmap": { 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgmap": { 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-30T09:25:13.995 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsmap": { 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": false, 2024-04-30T09:25:13.996 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modules": [ 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "iostat", 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "nfs", 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "restful" 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "servicemap": { 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modified": "2024-04-30T09:25:00.900690+0000", 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-30T09:25:13.997 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:13.998 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-30T09:25:13.998 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-30T09:25:14.569 INFO:teuthology.orchestra.run.smithi026.stderr:mgr not available, waiting (3/15)... 2024-04-30T09:25:14.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:14 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3677331383' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-30T09:25:14.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:14 smithi026 ceph-mon[22362]: mgrmap e3: y(active, since 1.0069s) 2024-04-30T09:25:17.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:17 smithi026 ceph-mon[22362]: mgrmap e4: y(active, since 2s) 2024-04-30T09:25:20.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:19 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4246267216' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-30T09:25:20.540 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:20.543 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsid": "703f4aea-06d3-11ef-bc93-c7b262605968", 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "health": { 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "checks": {}, 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mutes": [] 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum": [ 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 0 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:20.544 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "a" 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "quorum_age": 14, 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "monmap": { 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osdmap": { 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-30T09:25:20.545 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-30T09:25:20.546 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-30T09:25:20.546 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:20.546 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgmap": { 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "fsmap": { 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-30T09:25:20.549 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": true, 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modules": [ 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "iostat", 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "nfs", 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "restful" 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ], 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "servicemap": { 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-30T09:25:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "modified": "2024-04-30T09:25:00.900690+0000", 2024-04-30T09:25:20.551 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "services": {} 2024-04-30T09:25:20.551 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: }, 2024-04-30T09:25:20.551 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-30T09:25:20.551 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-30T09:25:24.835 INFO:teuthology.orchestra.run.smithi026.stderr:mgr is available 2024-04-30T09:25:25.429 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:25.429 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [global] 2024-04-30T09:25:25.429 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: fsid = 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:25:25.429 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-30T09:25:25.429 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-30T09:25:25.430 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-30T09:25:25.430 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-30T09:25:25.430 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:25.430 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [mgr] 2024-04-30T09:25:25.430 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-30T09:25:25.430 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 2024-04-30T09:25:25.430 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: [osd] 2024-04-30T09:25:25.430 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-30T09:25:25.430 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-30T09:25:25.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:25 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3884533800' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-30T09:25:25.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:25 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3884533800' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-30T09:25:25.991 INFO:teuthology.orchestra.run.smithi026.stderr:Enabling cephadm module... 2024-04-30T09:25:26.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:26 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4045588274' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-30T09:25:26.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ignoring --setuser ceph since I am not root 2024-04-30T09:25:26.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ignoring --setgroup ceph since I am not root 2024-04-30T09:25:26.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:26.853+0000 7f32c229a000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-30T09:25:27.270 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:26.986+0000 7f32c229a000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:25:27.270 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:27.088+0000 7f32c229a000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-30T09:25:27.522 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:27.408+0000 7f32c229a000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-30T09:25:27.522 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:27.494+0000 7f32c229a000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-30T09:25:27.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:27 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4045588274' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-30T09:25:27.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:27 smithi026 ceph-mon[22362]: mgrmap e5: y(active, since 13s) 2024-04-30T09:25:28.128 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-30T09:25:28.128 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-30T09:25:28.128 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": true, 2024-04-30T09:25:28.128 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-30T09:25:28.128 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-30T09:25:28.128 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-30T09:25:28.691 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:28 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:28.678+0000 7f32c229a000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-30T09:25:28.712 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for the mgr to restart... 2024-04-30T09:25:28.712 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mgr epoch 5... 2024-04-30T09:25:28.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:28 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1624529856' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-30T09:25:28.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:28 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:28.851+0000 7f32c229a000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-30T09:25:28.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:28 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:28.936+0000 7f32c229a000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-30T09:25:29.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:29.262+0000 7f32c229a000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-30T09:25:29.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:29.356+0000 7f32c229a000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-30T09:25:29.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:29.454+0000 7f32c229a000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-30T09:25:30.167 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:29.711+0000 7f32c229a000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:25:30.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:30.166+0000 7f32c229a000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-30T09:25:30.778 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:30.678+0000 7f32c229a000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-30T09:25:31.086 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:30.777+0000 7f32c229a000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-30T09:25:31.086 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:31.085+0000 7f32c229a000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-30T09:25:31.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:31.177+0000 7f32c229a000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-30T09:25:31.732 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:31.594+0000 7f32c229a000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-30T09:25:32.034 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:31.731+0000 7f32c229a000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-30T09:25:32.035 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:31.825+0000 7f32c229a000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-30T09:25:32.035 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:31.942+0000 7f32c229a000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-30T09:25:32.035 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:32.033+0000 7f32c229a000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-30T09:25:32.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:32.598+0000 7f32c229a000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-30T09:25:33.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:32.969+0000 7f32c229a000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-30T09:25:34.034 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-30T09:25:34.034 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-30T09:25:34.035 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "initialized": true 2024-04-30T09:25:34.035 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-30T09:25:34.332 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:25:34] ENGINE Bus STARTING 2024-04-30T09:25:34.332 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:25:34] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:25:34.332 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:25:34] ENGINE Bus STARTED 2024-04-30T09:25:34.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: Active manager daemon y restarted 2024-04-30T09:25:34.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: Activating manager daemon y 2024-04-30T09:25:34.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: osdmap e2: 0 total, 0 up, 0 in 2024-04-30T09:25:34.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: mgrmap e6: y(active, starting, since 0.0525305s) 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: Manager daemon y is now available 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:25:34.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:33 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:25:34.601 INFO:teuthology.orchestra.run.smithi026.stderr:mgr epoch 5 is available 2024-04-30T09:25:34.602 INFO:teuthology.orchestra.run.smithi026.stderr:Setting orchestrator backend to cephadm... 2024-04-30T09:25:35.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:35 smithi026 ceph-mon[22362]: mgrmap e7: y(active, since 1.0551s) 2024-04-30T09:25:35.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:35 smithi026 ceph-mon[22362]: [30/Apr/2024:09:25:34] ENGINE Bus STARTING 2024-04-30T09:25:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:35 smithi026 ceph-mon[22362]: [30/Apr/2024:09:25:34] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:25:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:35 smithi026 ceph-mon[22362]: [30/Apr/2024:09:25:34] ENGINE Bus STARTED 2024-04-30T09:25:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:35 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:35 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:25:36.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:36 smithi026 ceph-mon[22362]: mgrmap e8: y(active, since 2s) 2024-04-30T09:25:36.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:36 smithi026 ceph-mon[22362]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:25:36.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:36 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:36.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:36 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:25:36.431 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: value unchanged 2024-04-30T09:25:37.148 INFO:teuthology.orchestra.run.smithi026.stderr:Generating ssh key... 2024-04-30T09:25:37.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:37 smithi026 ceph-mon[22362]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:25:38.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: Generating public/private rsa key pair. 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: Your identification has been saved in /tmp/tmpddvi3ebz/key. 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: Your public key has been saved in /tmp/tmpddvi3ebz/key.pub. 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: The key fingerprint is: 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: SHA256:Ktvw+ZBbp+J2aWlfjHhqBemMp1AottT0DWlpXorB0yk ceph-703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: The key's randomart image is: 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: +---[RSA 3072]----+ 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: | . . + | 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: | E O . | 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: | o @ = . | 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: | + + = + | 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: | o o . +S. | 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: | . . .o+..o | 2024-04-30T09:25:38.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: | o.+oo++ o | 2024-04-30T09:25:38.698 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: | *+=B= . | 2024-04-30T09:25:38.698 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: | .o*B+.. | 2024-04-30T09:25:38.698 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:38 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: +----[SHA256]-----+ 2024-04-30T09:25:39.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:39 smithi026 ceph-mon[22362]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:25:39.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:39 smithi026 ceph-mon[22362]: Generating ssh key... 2024-04-30T09:25:39.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:39 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:39.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:39 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:39.449 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDPVFOPKsLLWMy8HEqWXKjr/oKzqgdxZ5NG9ZythfB9C60XZZGvp7ZnNiPDS+tzX/jPvpQhncQ+oK+qYTTLt5KrCyKKOtS/WAGz9HvGihjh69JjBYjsf87Z1SgJyqxOLQCRe7ZwCdNHwhr9O/xfpGwJf8K+1qWiUQXH7vGGbvIHeENEasW0QEvENEnbdqGXTBbej7NkXheKf0akCbCgnUCq4Q+hkrWpOGz7JJO3y0vmu7RxhopUbhfNMPLcPdX/6s6dv93D42wqgBVLJQ7ulfco0jhMZv8QLdhiXNzIC/Qq/FWQte6ghXMe9OF5IutbDocIBATwD7XvIEG+TXFO7By2UaYj0GjmjzLw+P7VapdDs7sqnbg3zH2qyKcXhz3PyOHr+Lq7sjfGOmBeJLJxPetkUiC1MxT63oC8GETvzExLWiTghhBLM2XEVdSSbXORD1uFofSAt5k6nKxloyDmiaCHgYZKWL+x3GjhSplWJws73vm7OfMkiz2MVyY8x1KeQZ8= ceph-703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:25:40.027 INFO:teuthology.orchestra.run.smithi026.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-30T09:25:40.028 INFO:teuthology.orchestra.run.smithi026.stderr:Adding key to root@localhost authorized_keys... 2024-04-30T09:25:40.028 INFO:teuthology.orchestra.run.smithi026.stderr:Adding host smithi026... 2024-04-30T09:25:40.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:40 smithi026 ceph-mon[22362]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:25:41.521 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: Added host 'smithi026' with addr '172.21.15.26' 2024-04-30T09:25:41.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:41 smithi026 ceph-mon[22362]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi026", "addr": "172.21.15.26", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:25:42.092 INFO:teuthology.orchestra.run.smithi026.stderr:Deploying unmanaged mon service... 2024-04-30T09:25:42.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:42 smithi026 ceph-mon[22362]: Deploying cephadm binary to smithi026 2024-04-30T09:25:42.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:42 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:42.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:42 smithi026 ceph-mon[22362]: Added host smithi026 2024-04-30T09:25:42.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:42 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:25:42.766 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-30T09:25:43.310 INFO:teuthology.orchestra.run.smithi026.stderr:Deploying unmanaged mgr service... 2024-04-30T09:25:43.894 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:43 smithi026 ceph-mon[22362]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:25:43.895 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:43 smithi026 ceph-mon[22362]: Saving service mon spec with placement count:5 2024-04-30T09:25:43.895 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:43 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:43.950 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-30T09:25:45.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:44 smithi026 ceph-mon[22362]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:25:45.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:44 smithi026 ceph-mon[22362]: Saving service mgr spec with placement count:2 2024-04-30T09:25:45.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:44 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:46.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:46 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:46.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:46 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:46.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:46 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2119663612' entity='client.admin' 2024-04-30T09:25:48.377 INFO:teuthology.orchestra.run.smithi026.stderr:Enabling the dashboard module... 2024-04-30T09:25:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:48 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:48 smithi026 ceph-mon[22362]: from='mgr.14120 172.21.15.26:0/2751984446' entity='mgr.y' 2024-04-30T09:25:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:48 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1760985732' entity='client.admin' 2024-04-30T09:25:49.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:49 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3229428860' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-30T09:25:49.339 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ignoring --setuser ceph since I am not root 2024-04-30T09:25:49.339 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ignoring --setgroup ceph since I am not root 2024-04-30T09:25:49.339 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:49.337+0000 7f98b9810000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-30T09:25:49.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:49.471+0000 7f98b9810000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:25:49.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:49.572+0000 7f98b9810000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-30T09:25:50.088 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:49.891+0000 7f98b9810000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-30T09:25:50.088 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:49.976+0000 7f98b9810000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-30T09:25:50.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:50 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3229428860' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-30T09:25:50.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:50 smithi026 ceph-mon[22362]: mgrmap e9: y(active, since 16s) 2024-04-30T09:25:50.746 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-30T09:25:50.746 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-30T09:25:50.746 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "available": true, 2024-04-30T09:25:50.746 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "active_name": "y", 2024-04-30T09:25:50.746 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-30T09:25:50.746 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-30T09:25:51.434 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:51 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:51.177+0000 7f98b9810000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-30T09:25:51.434 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:51 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:51.348+0000 7f98b9810000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-30T09:25:51.434 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:51 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:51.433+0000 7f98b9810000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-30T09:25:51.542 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for the mgr to restart... 2024-04-30T09:25:51.543 INFO:teuthology.orchestra.run.smithi026.stderr:Waiting for mgr epoch 9... 2024-04-30T09:25:51.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:51 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3648967791' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-30T09:25:52.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:51 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:51.765+0000 7f98b9810000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-30T09:25:52.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:51 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:51.860+0000 7f98b9810000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-30T09:25:52.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:51 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:51.959+0000 7f98b9810000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-30T09:25:52.669 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:52.210+0000 7f98b9810000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:25:52.669 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:52.668+0000 7f98b9810000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-30T09:25:53.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:53.172+0000 7f98b9810000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-30T09:25:53.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:53.268+0000 7f98b9810000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-30T09:25:53.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:53.574+0000 7f98b9810000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-30T09:25:53.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:53.666+0000 7f98b9810000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-30T09:25:54.435 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:54.084+0000 7f98b9810000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-30T09:25:54.435 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:54.221+0000 7f98b9810000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-30T09:25:54.435 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:54.316+0000 7f98b9810000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-30T09:25:54.435 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:54.434+0000 7f98b9810000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-30T09:25:54.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:54.526+0000 7f98b9810000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-30T09:25:55.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:55 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:55.095+0000 7f98b9810000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-30T09:25:55.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:55 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:25:55.468+0000 7f98b9810000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-30T09:25:56.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:56 smithi026 ceph-mon[22362]: Active manager daemon y restarted 2024-04-30T09:25:56.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:56 smithi026 ceph-mon[22362]: Activating manager daemon y 2024-04-30T09:25:56.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:56 smithi026 ceph-mon[22362]: osdmap e3: 0 total, 0 up, 0 in 2024-04-30T09:25:56.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:56 smithi026 ceph-mon[22362]: mgrmap e10: y(active, starting, since 0.995938s) 2024-04-30T09:25:56.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:25:56.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:25:56.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:25:56.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:25:56.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:25:57.422 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:25:57] ENGINE Bus STARTING 2024-04-30T09:25:57.422 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:25:57] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:25:57.422 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:25:57] ENGINE Bus STARTED 2024-04-30T09:25:57.484 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: { 2024-04-30T09:25:57.485 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-30T09:25:57.485 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: "initialized": true 2024-04-30T09:25:57.485 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: } 2024-04-30T09:25:57.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-mon[22362]: Manager daemon y is now available 2024-04-30T09:25:57.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:25:57.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:25:57.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:25:57.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:25:57.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-mon[22362]: [30/Apr/2024:09:25:57] ENGINE Bus STARTING 2024-04-30T09:25:57.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:25:58.232 INFO:teuthology.orchestra.run.smithi026.stderr:mgr epoch 9 is available 2024-04-30T09:25:58.232 INFO:teuthology.orchestra.run.smithi026.stderr:Generating a dashboard self-signed certificate... 2024-04-30T09:25:58.519 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:58 smithi026 ceph-mon[22362]: [30/Apr/2024:09:25:57] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:25:58.519 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:58 smithi026 ceph-mon[22362]: [30/Apr/2024:09:25:57] ENGINE Bus STARTED 2024-04-30T09:25:58.519 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:58 smithi026 ceph-mon[22362]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-30T09:25:58.519 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:58 smithi026 ceph-mon[22362]: mgrmap e11: y(active, since 2s) 2024-04-30T09:25:58.519 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:58 smithi026 ceph-mon[22362]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-30T09:25:58.519 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:58 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:25:59.107 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-30T09:25:59.775 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:59 smithi026 ceph-mon[22362]: mgrmap e12: y(active, since 3s) 2024-04-30T09:25:59.775 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:59 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:25:59.775 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:59 smithi026 ceph-mon[22362]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:25:59.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:59 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:25:59.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:25:59 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:25:59.778 INFO:teuthology.orchestra.run.smithi026.stderr:Creating initial admin user... 2024-04-30T09:26:01.575 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$OrUelS0EaJfVLkNhOSYmbuS7fdJaiIU4ZuXyWhqlClAFssMShD/li", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1714469161, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-30T09:26:02.265 INFO:teuthology.orchestra.run.smithi026.stderr:Fetching dashboard port number... 2024-04-30T09:26:02.920 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:02 smithi026 ceph-mon[22362]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:26:02.920 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:02 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:02.921 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: 8443 2024-04-30T09:26:03.603 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: disabled 2024-04-30T09:26:03.610 INFO:teuthology.orchestra.run.smithi026.stderr:systemctl: inactive 2024-04-30T09:26:03.611 INFO:teuthology.orchestra.run.smithi026.stderr:firewalld.service is not enabled 2024-04-30T09:26:03.612 INFO:teuthology.orchestra.run.smithi026.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-30T09:26:03.614 INFO:teuthology.orchestra.run.smithi026.stderr:Ceph Dashboard is now available at: 2024-04-30T09:26:03.614 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T09:26:03.614 INFO:teuthology.orchestra.run.smithi026.stderr: URL: https://smithi026:8443/ 2024-04-30T09:26:03.614 INFO:teuthology.orchestra.run.smithi026.stderr: User: admin 2024-04-30T09:26:03.614 INFO:teuthology.orchestra.run.smithi026.stderr: Password: le7f7cluq5 2024-04-30T09:26:03.614 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T09:26:03.614 INFO:teuthology.orchestra.run.smithi026.stderr:Enabling autotune for osd_memory_target 2024-04-30T09:26:03.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:03 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/679961410' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-30T09:26:04.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:04 smithi026 ceph-mon[22362]: mgrmap e13: y(active, since 8s) 2024-04-30T09:26:05.708 INFO:teuthology.orchestra.run.smithi026.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-30T09:26:06.336 INFO:teuthology.orchestra.run.smithi026.stderr:You can access the Ceph CLI with: 2024-04-30T09:26:06.336 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T09:26:06.336 INFO:teuthology.orchestra.run.smithi026.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr: ceph telemetry on 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr:For more information see: 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T09:26:06.337 INFO:teuthology.orchestra.run.smithi026.stderr:Bootstrap complete. 2024-04-30T09:26:06.375 INFO:tasks.cephadm:Fetching config... 2024-04-30T09:26:06.375 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:26:06.375 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-30T09:26:06.391 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-30T09:26:06.392 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:26:06.392 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-30T09:26:06.448 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-30T09:26:06.449 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:26:06.449 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-30T09:26:06.514 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-30T09:26:06.514 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:26:06.514 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-30T09:26:06.570 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-30T09:26:06.570 DEBUG:teuthology.orchestra.run.smithi026:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDPVFOPKsLLWMy8HEqWXKjr/oKzqgdxZ5NG9ZythfB9C60XZZGvp7ZnNiPDS+tzX/jPvpQhncQ+oK+qYTTLt5KrCyKKOtS/WAGz9HvGihjh69JjBYjsf87Z1SgJyqxOLQCRe7ZwCdNHwhr9O/xfpGwJf8K+1qWiUQXH7vGGbvIHeENEasW0QEvENEnbdqGXTBbej7NkXheKf0akCbCgnUCq4Q+hkrWpOGz7JJO3y0vmu7RxhopUbhfNMPLcPdX/6s6dv93D42wqgBVLJQ7ulfco0jhMZv8QLdhiXNzIC/Qq/FWQte6ghXMe9OF5IutbDocIBATwD7XvIEG+TXFO7By2UaYj0GjmjzLw+P7VapdDs7sqnbg3zH2qyKcXhz3PyOHr+Lq7sjfGOmBeJLJxPetkUiC1MxT63oC8GETvzExLWiTghhBLM2XEVdSSbXORD1uFofSAt5k6nKxloyDmiaCHgYZKWL+x3GjhSplWJws73vm7OfMkiz2MVyY8x1KeQZ8= ceph-703f4aea-06d3-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-30T09:26:06.644 INFO:teuthology.orchestra.run.smithi026.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDPVFOPKsLLWMy8HEqWXKjr/oKzqgdxZ5NG9ZythfB9C60XZZGvp7ZnNiPDS+tzX/jPvpQhncQ+oK+qYTTLt5KrCyKKOtS/WAGz9HvGihjh69JjBYjsf87Z1SgJyqxOLQCRe7ZwCdNHwhr9O/xfpGwJf8K+1qWiUQXH7vGGbvIHeENEasW0QEvENEnbdqGXTBbej7NkXheKf0akCbCgnUCq4Q+hkrWpOGz7JJO3y0vmu7RxhopUbhfNMPLcPdX/6s6dv93D42wqgBVLJQ7ulfco0jhMZv8QLdhiXNzIC/Qq/FWQte6ghXMe9OF5IutbDocIBATwD7XvIEG+TXFO7By2UaYj0GjmjzLw+P7VapdDs7sqnbg3zH2qyKcXhz3PyOHr+Lq7sjfGOmBeJLJxPetkUiC1MxT63oC8GETvzExLWiTghhBLM2XEVdSSbXORD1uFofSAt5k6nKxloyDmiaCHgYZKWL+x3GjhSplWJws73vm7OfMkiz2MVyY8x1KeQZ8= ceph-703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:26:06.655 DEBUG:teuthology.orchestra.run.smithi046:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDPVFOPKsLLWMy8HEqWXKjr/oKzqgdxZ5NG9ZythfB9C60XZZGvp7ZnNiPDS+tzX/jPvpQhncQ+oK+qYTTLt5KrCyKKOtS/WAGz9HvGihjh69JjBYjsf87Z1SgJyqxOLQCRe7ZwCdNHwhr9O/xfpGwJf8K+1qWiUQXH7vGGbvIHeENEasW0QEvENEnbdqGXTBbej7NkXheKf0akCbCgnUCq4Q+hkrWpOGz7JJO3y0vmu7RxhopUbhfNMPLcPdX/6s6dv93D42wqgBVLJQ7ulfco0jhMZv8QLdhiXNzIC/Qq/FWQte6ghXMe9OF5IutbDocIBATwD7XvIEG+TXFO7By2UaYj0GjmjzLw+P7VapdDs7sqnbg3zH2qyKcXhz3PyOHr+Lq7sjfGOmBeJLJxPetkUiC1MxT63oC8GETvzExLWiTghhBLM2XEVdSSbXORD1uFofSAt5k6nKxloyDmiaCHgYZKWL+x3GjhSplWJws73vm7OfMkiz2MVyY8x1KeQZ8= ceph-703f4aea-06d3-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-30T09:26:06.697 INFO:teuthology.orchestra.run.smithi046.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDPVFOPKsLLWMy8HEqWXKjr/oKzqgdxZ5NG9ZythfB9C60XZZGvp7ZnNiPDS+tzX/jPvpQhncQ+oK+qYTTLt5KrCyKKOtS/WAGz9HvGihjh69JjBYjsf87Z1SgJyqxOLQCRe7ZwCdNHwhr9O/xfpGwJf8K+1qWiUQXH7vGGbvIHeENEasW0QEvENEnbdqGXTBbej7NkXheKf0akCbCgnUCq4Q+hkrWpOGz7JJO3y0vmu7RxhopUbhfNMPLcPdX/6s6dv93D42wqgBVLJQ7ulfco0jhMZv8QLdhiXNzIC/Qq/FWQte6ghXMe9OF5IutbDocIBATwD7XvIEG+TXFO7By2UaYj0GjmjzLw+P7VapdDs7sqnbg3zH2qyKcXhz3PyOHr+Lq7sjfGOmBeJLJxPetkUiC1MxT63oC8GETvzExLWiTghhBLM2XEVdSSbXORD1uFofSAt5k6nKxloyDmiaCHgYZKWL+x3GjhSplWJws73vm7OfMkiz2MVyY8x1KeQZ8= ceph-703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:26:06.709 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-30T09:26:06.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:06 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2109829458' entity='client.admin' 2024-04-30T09:26:09.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:09 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/419027026' entity='client.admin' 2024-04-30T09:26:10.100 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-30T09:26:10.101 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-30T09:26:12.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:11 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:12.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:11 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi026", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:26:12.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:11 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:12.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:11 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:12.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:11 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:13.342 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi046 2024-04-30T09:26:13.343 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-04-30T09:26:13.343 DEBUG:teuthology.orchestra.run.smithi046:> dd of=/etc/ceph/ceph.conf 2024-04-30T09:26:13.363 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-04-30T09:26:13.363 DEBUG:teuthology.orchestra.run.smithi046:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:26:13.421 INFO:tasks.cephadm:Adding host smithi046 to orchestrator... 2024-04-30T09:26:13.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch host add smithi046 2024-04-30T09:26:13.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:13 smithi026 ceph-mon[22362]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:26:13.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:13.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:13.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:13.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:13.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:13 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:26:13.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:13 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:26:13.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:13.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:16.156 INFO:teuthology.orchestra.run.smithi026.stdout:Added host 'smithi046' with addr '172.21.15.46' 2024-04-30T09:26:16.758 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-30T09:26:17.409 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:17 smithi026 ceph-mon[22362]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi046", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:26:17.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:17 smithi026 ceph-mon[22362]: Deploying cephadm binary to smithi046 2024-04-30T09:26:17.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:17.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:17 smithi026 ceph-mon[22362]: Added host smithi046 2024-04-30T09:26:17.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:17.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:17.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:17.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:17.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:18.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:18 smithi026 ceph-mon[22362]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:18.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:18 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:18.690 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:26:18.690 INFO:teuthology.orchestra.run.smithi026.stdout:[{"addr": "172.21.15.26", "hostname": "smithi026", "labels": [], "status": ""}, {"addr": "172.21.15.46", "hostname": "smithi046", "labels": [], "status": ""}] 2024-04-30T09:26:19.316 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-30T09:26:19.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-30T09:26:20.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:20 smithi026 ceph-mon[22362]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:20.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:20 smithi026 ceph-mon[22362]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-30T09:26:21.474 INFO:teuthology.orchestra.run.smithi026.stderr:adjusted tunables profile to default 2024-04-30T09:26:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi046", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:26:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:21 smithi026 ceph-mon[22362]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:21 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2133644576' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-30T09:26:22.070 INFO:tasks.cephadm:Adding mon.a on smithi026 2024-04-30T09:26:22.070 INFO:tasks.cephadm:Adding mon.c on smithi026 2024-04-30T09:26:22.070 INFO:tasks.cephadm:Adding mon.b on smithi046 2024-04-30T09:26:22.070 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch apply mon '3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi046:172.21.15.46=b' 2024-04-30T09:26:22.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:22 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2133644576' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-30T09:26:22.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:22 smithi026 ceph-mon[22362]: osdmap e4: 0 total, 0 up, 0 in 2024-04-30T09:26:23.866 INFO:teuthology.orchestra.run.smithi046.stdout:Scheduled mon update... 2024-04-30T09:26:23.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:23 smithi026 ceph-mon[22362]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:24.568 DEBUG:teuthology.orchestra.run.smithi026:mon.c> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.c.service 2024-04-30T09:26:24.570 DEBUG:teuthology.orchestra.run.smithi046:mon.b> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.b.service 2024-04-30T09:26:24.573 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-30T09:26:24.573 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-30T09:26:25.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi046:172.21.15.46=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi046:172.21.15.46=b;count:3 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:26:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:25.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:25 smithi026 ceph-mon[22362]: Deploying daemon mon.c on smithi026 2024-04-30T09:26:25.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:25 smithi026 ceph-mon[22362]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:26.518 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:26:26.518 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":1,"fsid":"703f4aea-06d3-11ef-bc93-c7b262605968","modified":"2024-04-30T09:24:58.133666Z","created":"2024-04-30T09:24:58.133666Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-30T09:26:26.520 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 1 2024-04-30T09:26:27.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:26 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/3232281403' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-30T09:26:28.046 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-30T09:26:28.047 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-30T09:26:32.920 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:26:32.920 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":2,"fsid":"703f4aea-06d3-11ef-bc93-c7b262605968","modified":"2024-04-30T09:26:27.542935Z","created":"2024-04-30T09:24:58.133666Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3301","nonce":0},{"type":"v1","addr":"172.21.15.26:6790","nonce":0}]},"addr":"172.21.15.26:6790/0","public_addr":"172.21.15.26:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-30T09:26:32.922 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 2 2024-04-30T09:26:32.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: mon.a calling monitor election 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: mon.c calling monitor election 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:32.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-30T09:26:32.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:26:32.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: fsmap 2024-04-30T09:26:32.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: osdmap e4: 0 total, 0 up, 0 in 2024-04-30T09:26:32.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: mgrmap e13: y(active, since 37s) 2024-04-30T09:26:32.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: overall HEALTH_OK 2024-04-30T09:26:32.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:32.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:26:32.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:33.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:33 smithi026 ceph-mon[27123]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:33.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:33 smithi026 ceph-mon[27123]: Deploying daemon mon.b on smithi046 2024-04-30T09:26:33.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:33 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/967925729' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-30T09:26:33.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:33 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:33 smithi026 ceph-mon[22362]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:33 smithi026 ceph-mon[22362]: Deploying daemon mon.b on smithi046 2024-04-30T09:26:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:33 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/967925729' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-30T09:26:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:33 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:34.599 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-30T09:26:34.599 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-30T09:26:35.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:35 smithi026 ceph-mon[22362]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:35.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:35 smithi026 ceph-mon[27123]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:36.525 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:36 smithi046 systemd[1]: Starting Ceph mon.b for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:26:37.238 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:26:37.238 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":2,"fsid":"703f4aea-06d3-11ef-bc93-c7b262605968","modified":"2024-04-30T09:26:27.542935Z","created":"2024-04-30T09:24:58.133666Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3301","nonce":0},{"type":"v1","addr":"172.21.15.26:6790","nonce":0}]},"addr":"172.21.15.26:6790/0","public_addr":"172.21.15.26:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-30T09:26:37.239 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 2 2024-04-30T09:26:37.315 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 podman[23138]: 2024-04-30 09:26:36.957330551 +0000 UTC m=+0.147750863 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-30T09:26:37.315 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 podman[23138]: 2024-04-30 09:26:37.053547539 +0000 UTC m=+0.243967836 container create e7247b5c791f3583e5b0596c03aa606e2a00412f99e17784a1d6a833bab2c4f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, io.openshift.tags=base centos centos-stream, architecture=x86_64, GIT_CLEAN=True, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, com.redhat.component=centos-stream-container, RELEASE=HEAD, vendor=Red Hat, Inc., GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, ceph=True, io.k8s.display-name=CentOS Stream 8, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-30T09:26:37.315 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 podman[23138]: 2024-04-30 09:26:37.239395146 +0000 UTC m=+0.429815445 container init e7247b5c791f3583e5b0596c03aa606e2a00412f99e17784a1d6a833bab2c4f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, io.openshift.expose-services=, vendor=Red Hat, Inc., vcs-type=git, io.openshift.tags=base centos centos-stream, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, com.redhat.component=centos-stream-container, ceph=True, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_REPO=https://github.com/ceph/ceph-container.git, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754) 2024-04-30T09:26:37.315 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 podman[23138]: 2024-04-30 09:26:37.243002712 +0000 UTC m=+0.433423011 container start e7247b5c791f3583e5b0596c03aa606e2a00412f99e17784a1d6a833bab2c4f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., io.buildah.version=1.19.8, vcs-type=git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, version=8, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, GIT_CLEAN=True, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-30T09:26:37.315 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: set uid:gid to 167:167 (ceph:ceph) 2024-04-30T09:26:37.315 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-30T09:26:37.315 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: pidfile_write: ignore empty --pid-file 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: load: jerasure load: lrc 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: RocksDB version: 6.15.5 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Compile date Apr 18 2022 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: DB SUMMARY 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: DB Session ID: X5I4TZDP64696J95WBXN 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: CURRENT file: CURRENT 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: IDENTITY file: IDENTITY 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 636 ; 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.error_if_exists: 0 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.create_if_missing: 0 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.paranoid_checks: 1 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-30T09:26:37.316 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.env: 0x558d018c6860 2024-04-30T09:26:37.317 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.fs: Posix File System 2024-04-30T09:26:37.317 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.info_log: 0x558d02537dc0 2024-04-30T09:26:37.317 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_file_opening_threads: 16 2024-04-30T09:26:37.317 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.statistics: (nil) 2024-04-30T09:26:37.317 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.use_fsync: 0 2024-04-30T09:26:37.318 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_log_file_size: 0 2024-04-30T09:26:37.318 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-30T09:26:37.318 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-30T09:26:37.318 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.keep_log_file_num: 1000 2024-04-30T09:26:37.318 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.recycle_log_file_num: 0 2024-04-30T09:26:37.318 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.allow_fallocate: 1 2024-04-30T09:26:37.318 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.allow_mmap_reads: 0 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.allow_mmap_writes: 0 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.use_direct_reads: 0 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.create_missing_column_families: 0 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.db_log_dir: 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.advise_random_on_open: 1 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.db_write_buffer_size: 0 2024-04-30T09:26:37.319 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.write_buffer_manager: 0x558d02628240 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.rate_limiter: (nil) 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.wal_recovery_mode: 2 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.enable_thread_tracking: 0 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.enable_pipelined_write: 0 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.unordered_write: 0 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.row_cache: None 2024-04-30T09:26:37.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.wal_filter: None 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.allow_ingest_behind: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.preserve_deletes: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.two_write_queues: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.manual_wal_flush: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.atomic_flush: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.log_readahead_size: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.best_efforts_recovery: 0 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-30T09:26:37.321 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.allow_data_in_errors: 0 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.db_host_id: __hostname__ 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_background_jobs: 2 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_background_compactions: -1 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_subcompactions: 1 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_total_wal_size: 0 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_open_files: -1 2024-04-30T09:26:37.322 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bytes_per_sync: 0 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_readahead_size: 0 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_background_flushes: -1 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Compression algorithms supported: 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: kZSTD supported: 0 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: kXpressCompression supported: 0 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: kLZ4HCCompression supported: 1 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: kLZ4Compression supported: 1 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: kBZip2Compression supported: 0 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: kZlibCompression supported: 1 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: kSnappyCompression supported: 1 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-04-30T09:26:37.323 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.merge_operator: 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_filter: None 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_filter_factory: None 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.sst_partitioner_factory: None 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558d02503d00) 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout: cache_index_and_filter_blocks: 1 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout: pin_top_level_index_and_filter: 1 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout: index_type: 0 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout: data_block_index_type: 0 2024-04-30T09:26:37.324 INFO:journalctl@ceph.mon.b.smithi046.stdout: index_shortening: 1 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: hash_index_allow_collision: 1 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: checksum: 1 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: no_block_cache: 0 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_cache: 0x558d0256e170 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_cache_name: BinnedLRUCache 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_cache_options: 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: capacity : 536870912 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: num_shard_bits : 4 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: strict_capacity_limit : 0 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: high_pri_pool_ratio: 0.000 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_cache_compressed: (nil) 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: persistent_cache: (nil) 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_size: 4096 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_size_deviation: 10 2024-04-30T09:26:37.325 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_restart_interval: 16 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: index_block_restart_interval: 1 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: metadata_block_size: 4096 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: partition_filters: 0 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: use_delta_encoding: 1 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: whole_key_filtering: 1 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: verify_compression: 0 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: read_amp_bytes_per_bit: 0 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: format_version: 4 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: enable_index_compression: 1 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_align: 0 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.write_buffer_size: 33554432 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_write_buffer_number: 2 2024-04-30T09:26:37.326 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compression: NoCompression 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bottommost_compression: Disabled 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.prefix_extractor: nullptr 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.num_levels: 7 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-30T09:26:37.327 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compression_opts.level: 32767 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compression_opts.strategy: 0 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compression_opts.enabled: false 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.target_file_size_base: 67108864 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-30T09:26:37.328 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.arena_block_size: 4194304 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.disable_auto_compactions: 0 2024-04-30T09:26:37.329 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.table_properties_collectors: 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.inplace_update_support: 0 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-30T09:26:37.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.bloom_locality: 0 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.max_successive_merges: 0 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.paranoid_file_checks: 0 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.force_consistency_checks: 1 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.report_bg_io_stats: 0 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.ttl: 2592000 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.enable_blob_files: false 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.min_blob_size: 0 2024-04-30T09:26:37.331 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.blob_file_size: 268435456 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 0 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/version_set.cc:4083] Creating manifest 7 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714469197271603, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #4 mode 2 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714469197271940, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1714469197, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b222fac2-6d29-4672-a965-4e9afe4dd1c1", "db_session_id": "X5I4TZDP64696J95WBXN"}} 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/version_set.cc:4083] Creating manifest 9 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714469197272274, "job": 1, "event": "recovery_finished"} 2024-04-30T09:26:37.332 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x558d02554a80 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: DB pointer 0x558d02564000 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** DB Stats ** 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** Compaction Stats [default] ** 2024-04-30T09:26:37.333 INFO:journalctl@ceph.mon.b.smithi046.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** Compaction Stats [default] ** 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-30T09:26:37.334 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: Cumulative compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: Interval compaction: 0.00 GB write, 0.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** Compaction Stats [default] ** 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** Compaction Stats [default] ** 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-30T09:26:37.335 INFO:journalctl@ceph.mon.b.smithi046.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.0 0.00 0.00 1 0.000 0 0 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b does not exist in monmap, will attempt to join an existing cluster 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: using public_addr v2:172.21.15.46:0/0 -> [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] 2024-04-30T09:26:37.336 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: starting mon.b rank -1 at public addrs [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] at bind addrs [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(???) e0 preinit fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).mds e1 new map 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).mds e1 print_map 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout: e1 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout: legacy client fscid: -1 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout: No filesystems configured 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-04-30T09:26:37.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.a calling monitor election 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:37.338 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.c calling monitor election 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: fsmap 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: osdmap e4: 0 total, 0 up, 0 in 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mgrmap e13: y(active, since 37s) 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: overall HEALTH_OK 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:37.339 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:37.340 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: Deploying daemon mon.b on smithi046 2024-04-30T09:26:37.340 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/967925729' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-30T09:26:37.340 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:37.340 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:37.340 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-30T09:26:37.340 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 ceph-mon[23181]: mon.b@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete,mon_cluster_log_to_file 2024-04-30T09:26:37.340 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 bash[23138]: e7247b5c791f3583e5b0596c03aa606e2a00412f99e17784a1d6a833bab2c4f7 2024-04-30T09:26:37.340 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:37 smithi046 systemd[1]: Started Ceph mon.b for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:26:38.853 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-30T09:26:38.853 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-30T09:26:42.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:26:42.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: mon.a calling monitor election 2024-04-30T09:26:42.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: mon.c calling monitor election 2024-04-30T09:26:42.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:42.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: fsmap 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: osdmap e4: 0 total, 0 up, 0 in 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: mgrmap e13: y(active, since 46s) 2024-04-30T09:26:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-30T09:26:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-30T09:26:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-30T09:26:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: mon.b (rank 2) addr [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] is down (out of quorum) 2024-04-30T09:26:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: mon.a calling monitor election 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: mon.c calling monitor election 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:42.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: fsmap 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: osdmap e4: 0 total, 0 up, 0 in 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: mgrmap e13: y(active, since 46s) 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-30T09:26:42.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: mon.b (rank 2) addr [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] is down (out of quorum) 2024-04-30T09:26:42.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:42.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:42.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:42.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:43.657 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:26:43.657 INFO:teuthology.orchestra.run.smithi046.stdout:{"epoch":3,"fsid":"703f4aea-06d3-11ef-bc93-c7b262605968","modified":"2024-04-30T09:26:37.291976Z","created":"2024-04-30T09:24:58.133666Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3301","nonce":0},{"type":"v1","addr":"172.21.15.26:6790","nonce":0}]},"addr":"172.21.15.26:6790/0","public_addr":"172.21.15.26:6790/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:3300","nonce":0},{"type":"v1","addr":"172.21.15.46:6789","nonce":0}]},"addr":"172.21.15.46:6789/0","public_addr":"172.21.15.46:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-30T09:26:43.659 INFO:teuthology.orchestra.run.smithi046.stderr:dumped monmap epoch 3 2024-04-30T09:26:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[22362]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:43.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[27123]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:43 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:44.223 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-30T09:26:44.223 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-30T09:26:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:26:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mon.a calling monitor election 2024-04-30T09:26:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mon.c calling monitor election 2024-04-30T09:26:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:26:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mon.a is new leader, mons a,c in quorum (ranks 0,1) 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: fsmap 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: osdmap e4: 0 total, 0 up, 0 in 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mgrmap e13: y(active, since 46s) 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,c 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: [WRN] MON_DOWN: 1/3 mons down, quorum a,c 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mon.b (rank 2) addr [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] is down (out of quorum) 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:44.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:44.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:44.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:44.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:44.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:44.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:44.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:44.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:44.900 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: mon.b calling monitor election 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: mon.b calling monitor election 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: mon.c calling monitor election 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: mon.a calling monitor election 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: fsmap 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: osdmap e4: 0 total, 0 up, 0 in 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: mgrmap e13: y(active, since 48s) 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: Cluster is now healthy 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[22362]: overall HEALTH_OK 2024-04-30T09:26:44.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: mon.b calling monitor election 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: mon.b calling monitor election 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: mon.c calling monitor election 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: mon.a calling monitor election 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: fsmap 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: osdmap e4: 0 total, 0 up, 0 in 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: mgrmap e13: y(active, since 48s) 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: Cluster is now healthy 2024-04-30T09:26:44.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:44 smithi026 ceph-mon[27123]: overall HEALTH_OK 2024-04-30T09:26:44.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mon.b calling monitor election 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mon.b calling monitor election 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mon.c calling monitor election 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mon.a calling monitor election 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: fsmap 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: osdmap e4: 0 total, 0 up, 0 in 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: mgrmap e13: y(active, since 48s) 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: Cluster is now healthy 2024-04-30T09:26:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:44 smithi046 ceph-mon[23181]: overall HEALTH_OK 2024-04-30T09:26:46.087 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[27123]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[27123]: Reconfiguring mon.c (monmap changed)... 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[27123]: Reconfiguring daemon mon.c on smithi026 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[22362]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[22362]: Reconfiguring mon.c (monmap changed)... 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:26:46.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:46.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[22362]: Reconfiguring daemon mon.c on smithi026 2024-04-30T09:26:46.089 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:45 smithi046 ceph-mon[23181]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:45 smithi046 ceph-mon[23181]: Reconfiguring mon.c (monmap changed)... 2024-04-30T09:26:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:26:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:26:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:45 smithi046 ceph-mon[23181]: Reconfiguring daemon mon.c on smithi026 2024-04-30T09:26:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:26:46.874 INFO:teuthology.orchestra.run.smithi026.stdout:# minimal ceph.conf for 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:26:46.874 INFO:teuthology.orchestra.run.smithi026.stdout:[global] 2024-04-30T09:26:46.874 INFO:teuthology.orchestra.run.smithi026.stdout: fsid = 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:26:46.874 INFO:teuthology.orchestra.run.smithi026.stdout: mon_host = [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] 2024-04-30T09:26:47.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:47 smithi026 ceph-mon[27123]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:47.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:47 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:47.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:47 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/745434215' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:47.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:47 smithi026 ceph-mon[22362]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:47.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:47 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:47.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:47 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/745434215' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:47.785 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:47 smithi046 ceph-mon[23181]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:47.785 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:47 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:47.785 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:47 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/745434215' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:48.124 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-30T09:26:48.124 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:26:48.125 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.conf 2024-04-30T09:26:48.153 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:26:48.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:26:48.219 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-04-30T09:26:48.220 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd of=/etc/ceph/ceph.conf 2024-04-30T09:26:48.245 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-04-30T09:26:48.245 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:26:48.312 INFO:tasks.cephadm:Adding mgr.y on smithi026 2024-04-30T09:26:48.312 INFO:tasks.cephadm:Adding mgr.x on smithi046 2024-04-30T09:26:48.312 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch apply mgr '2;smithi026=y;smithi046=x' 2024-04-30T09:26:49.039 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:49.039 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:48 smithi046 ceph-mon[23181]: Reconfiguring mon.b (monmap changed)... 2024-04-30T09:26:49.039 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:26:49.039 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:26:49.039 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:49.039 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:48 smithi046 ceph-mon[23181]: Reconfiguring daemon mon.b on smithi046 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[22362]: Reconfiguring mon.b (monmap changed)... 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[22362]: Reconfiguring daemon mon.b on smithi046 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[27123]: Reconfiguring mon.b (monmap changed)... 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:26:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:26:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:48 smithi026 ceph-mon[27123]: Reconfiguring daemon mon.b on smithi046 2024-04-30T09:26:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:50 smithi046 ceph-mon[23181]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:50 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:50 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:50 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:50 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:50 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[27123]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:50.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[22362]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:50 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:50.646 INFO:teuthology.orchestra.run.smithi046.stdout:Scheduled mgr update... 2024-04-30T09:26:51.182 DEBUG:teuthology.orchestra.run.smithi046:mgr.x> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.x.service 2024-04-30T09:26:51.184 INFO:tasks.cephadm:Deploying OSDs... 2024-04-30T09:26:51.185 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:26:51.185 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/scratch_devs of=/dev/stdout 2024-04-30T09:26:51.206 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-30T09:26:51.207 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_1 2024-04-30T09:26:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-30T09:26:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-30T09:26:51.265 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-30T09:26:51.265 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-30T09:26:51.265 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-04-30T09:26:51.265 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-04-30 09:26:27.044561451 +0000 2024-04-30T09:26:51.265 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-04-30 09:23:57.848744036 +0000 2024-04-30T09:26:51.265 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-04-30 09:23:57.848744036 +0000 2024-04-30T09:26:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-04-30 09:23:57.848744036 +0000 2024-04-30T09:26:51.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-30T09:26:51.333 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-04-30T09:26:51.333 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-04-30T09:26:51.333 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000129838 s, 3.9 MB/s 2024-04-30T09:26:51.334 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-30T09:26:51.394 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_2 2024-04-30T09:26:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-30T09:26:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-30T09:26:51.454 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-30T09:26:51.454 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-30T09:26:51.454 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-04-30T09:26:51.454 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-04-30 09:26:27.044561451 +0000 2024-04-30T09:26:51.454 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-04-30 09:23:58.198738626 +0000 2024-04-30T09:26:51.455 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-04-30 09:23:58.198738626 +0000 2024-04-30T09:26:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-04-30 09:23:58.198738626 +0000 2024-04-30T09:26:51.455 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-30T09:26:51.521 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-04-30T09:26:51.522 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-04-30T09:26:51.522 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000130952 s, 3.9 MB/s 2024-04-30T09:26:51.523 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-30T09:26:51.582 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_3 2024-04-30T09:26:51.639 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-30T09:26:51.639 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-30T09:26:51.639 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-04-30T09:26:51.640 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-30T09:26:51.640 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-04-30T09:26:51.640 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-04-30 09:26:27.045561436 +0000 2024-04-30T09:26:51.640 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-04-30 09:23:58.519733664 +0000 2024-04-30T09:26:51.640 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-04-30 09:23:58.519733664 +0000 2024-04-30T09:26:51.640 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-04-30 09:23:58.519733664 +0000 2024-04-30T09:26:51.640 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-30T09:26:51.705 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-04-30T09:26:51.705 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-04-30T09:26:51.705 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000118108 s, 4.3 MB/s 2024-04-30T09:26:51.706 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-30T09:26:51.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='client.24107 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi046=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: Saving service mgr spec with placement smithi026=y;smithi046=x;count:2 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:51.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:51 smithi046 ceph-mon[23181]: Deploying daemon mgr.x on smithi046 2024-04-30T09:26:51.763 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_4 2024-04-30T09:26:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-30T09:26:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-30T09:26:51.820 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 5h/5d Inode: 721 Links: 1 2024-04-30T09:26:51.820 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-30T09:26:51.820 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2024-04-30T09:26:51.820 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2024-04-30 09:26:27.045561436 +0000 2024-04-30T09:26:51.820 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2024-04-30 09:23:58.842728671 +0000 2024-04-30T09:26:51.820 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2024-04-30 09:23:58.842728671 +0000 2024-04-30T09:26:51.820 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: 2024-04-30 09:23:58.842728671 +0000 2024-04-30T09:26:51.821 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-30T09:26:51.883 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2024-04-30T09:26:51.883 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2024-04-30T09:26:51.883 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 9.8631e-05 s, 5.2 MB/s 2024-04-30T09:26:51.885 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-30T09:26:51.941 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-04-30T09:26:51.942 DEBUG:teuthology.orchestra.run.smithi046:> dd if=/scratch_devs of=/dev/stdout 2024-04-30T09:26:51.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:51.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='client.24107 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi046=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:26:51.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: Saving service mgr spec with placement smithi026=y;smithi046=x;count:2 2024-04-30T09:26:51.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:51.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[22362]: Deploying daemon mgr.x on smithi046 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='client.24107 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi046=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: Saving service mgr spec with placement smithi026=y;smithi046=x;count:2 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:26:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-30T09:26:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:26:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:51 smithi026 ceph-mon[27123]: Deploying daemon mgr.x on smithi046 2024-04-30T09:26:51.959 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-30T09:26:51.959 DEBUG:teuthology.orchestra.run.smithi046:> stat /dev/vg_nvme/lv_1 2024-04-30T09:26:52.016 INFO:teuthology.orchestra.run.smithi046.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-30T09:26:52.016 INFO:teuthology.orchestra.run.smithi046.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-30T09:26:52.016 INFO:teuthology.orchestra.run.smithi046.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-30T09:26:52.016 INFO:teuthology.orchestra.run.smithi046.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-30T09:26:52.016 INFO:teuthology.orchestra.run.smithi046.stdout:Context: system_u:object_r:device_t:s0 2024-04-30T09:26:52.016 INFO:teuthology.orchestra.run.smithi046.stdout:Access: 2024-04-30 09:26:36.316987310 +0000 2024-04-30T09:26:52.016 INFO:teuthology.orchestra.run.smithi046.stdout:Modify: 2024-04-30 09:24:07.427010148 +0000 2024-04-30T09:26:52.016 INFO:teuthology.orchestra.run.smithi046.stdout:Change: 2024-04-30 09:24:07.427010148 +0000 2024-04-30T09:26:52.016 INFO:teuthology.orchestra.run.smithi046.stdout: Birth: 2024-04-30 09:24:07.427010148 +0000 2024-04-30T09:26:52.016 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-30T09:26:52.081 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records in 2024-04-30T09:26:52.081 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records out 2024-04-30T09:26:52.081 INFO:teuthology.orchestra.run.smithi046.stderr:512 bytes copied, 0.000107088 s, 4.8 MB/s 2024-04-30T09:26:52.082 DEBUG:teuthology.orchestra.run.smithi046:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-30T09:26:52.139 DEBUG:teuthology.orchestra.run.smithi046:> stat /dev/vg_nvme/lv_2 2024-04-30T09:26:52.195 INFO:teuthology.orchestra.run.smithi046.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-30T09:26:52.195 INFO:teuthology.orchestra.run.smithi046.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-30T09:26:52.195 INFO:teuthology.orchestra.run.smithi046.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-30T09:26:52.195 INFO:teuthology.orchestra.run.smithi046.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-30T09:26:52.195 INFO:teuthology.orchestra.run.smithi046.stdout:Context: system_u:object_r:device_t:s0 2024-04-30T09:26:52.195 INFO:teuthology.orchestra.run.smithi046.stdout:Access: 2024-04-30 09:26:36.317987300 +0000 2024-04-30T09:26:52.195 INFO:teuthology.orchestra.run.smithi046.stdout:Modify: 2024-04-30 09:24:07.766006638 +0000 2024-04-30T09:26:52.195 INFO:teuthology.orchestra.run.smithi046.stdout:Change: 2024-04-30 09:24:07.766006638 +0000 2024-04-30T09:26:52.195 INFO:teuthology.orchestra.run.smithi046.stdout: Birth: 2024-04-30 09:24:07.766006638 +0000 2024-04-30T09:26:52.195 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-30T09:26:52.259 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records in 2024-04-30T09:26:52.259 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records out 2024-04-30T09:26:52.259 INFO:teuthology.orchestra.run.smithi046.stderr:512 bytes copied, 9.907e-05 s, 5.2 MB/s 2024-04-30T09:26:52.260 DEBUG:teuthology.orchestra.run.smithi046:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-30T09:26:52.317 DEBUG:teuthology.orchestra.run.smithi046:> stat /dev/vg_nvme/lv_3 2024-04-30T09:26:52.372 INFO:teuthology.orchestra.run.smithi046.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-30T09:26:52.373 INFO:teuthology.orchestra.run.smithi046.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-30T09:26:52.373 INFO:teuthology.orchestra.run.smithi046.stdout:Device: 5h/5d Inode: 703 Links: 1 2024-04-30T09:26:52.373 INFO:teuthology.orchestra.run.smithi046.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-30T09:26:52.373 INFO:teuthology.orchestra.run.smithi046.stdout:Context: system_u:object_r:device_t:s0 2024-04-30T09:26:52.373 INFO:teuthology.orchestra.run.smithi046.stdout:Access: 2024-04-30 09:26:36.317987300 +0000 2024-04-30T09:26:52.373 INFO:teuthology.orchestra.run.smithi046.stdout:Modify: 2024-04-30 09:24:08.099003191 +0000 2024-04-30T09:26:52.373 INFO:teuthology.orchestra.run.smithi046.stdout:Change: 2024-04-30 09:24:08.099003191 +0000 2024-04-30T09:26:52.373 INFO:teuthology.orchestra.run.smithi046.stdout: Birth: 2024-04-30 09:24:08.099003191 +0000 2024-04-30T09:26:52.373 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-30T09:26:52.557 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records in 2024-04-30T09:26:52.557 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records out 2024-04-30T09:26:52.557 INFO:teuthology.orchestra.run.smithi046.stderr:512 bytes copied, 0.000110438 s, 4.6 MB/s 2024-04-30T09:26:52.558 DEBUG:teuthology.orchestra.run.smithi046:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-30T09:26:52.575 DEBUG:teuthology.orchestra.run.smithi046:> stat /dev/vg_nvme/lv_4 2024-04-30T09:26:52.635 INFO:teuthology.orchestra.run.smithi046.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-30T09:26:52.635 INFO:teuthology.orchestra.run.smithi046.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-30T09:26:52.635 INFO:teuthology.orchestra.run.smithi046.stdout:Device: 5h/5d Inode: 722 Links: 1 2024-04-30T09:26:52.635 INFO:teuthology.orchestra.run.smithi046.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-30T09:26:52.635 INFO:teuthology.orchestra.run.smithi046.stdout:Context: system_u:object_r:device_t:s0 2024-04-30T09:26:52.635 INFO:teuthology.orchestra.run.smithi046.stdout:Access: 2024-04-30 09:26:52.511814469 +0000 2024-04-30T09:26:52.635 INFO:teuthology.orchestra.run.smithi046.stdout:Modify: 2024-04-30 09:24:08.453999515 +0000 2024-04-30T09:26:52.635 INFO:teuthology.orchestra.run.smithi046.stdout:Change: 2024-04-30 09:24:08.453999515 +0000 2024-04-30T09:26:52.635 INFO:teuthology.orchestra.run.smithi046.stdout: Birth: 2024-04-30 09:24:08.453999515 +0000 2024-04-30T09:26:52.636 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-30T09:26:52.740 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records in 2024-04-30T09:26:52.740 INFO:teuthology.orchestra.run.smithi046.stderr:1+0 records out 2024-04-30T09:26:52.740 INFO:teuthology.orchestra.run.smithi046.stderr:512 bytes copied, 0.000108404 s, 4.7 MB/s 2024-04-30T09:26:52.742 DEBUG:teuthology.orchestra.run.smithi046:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-30T09:26:52.757 INFO:tasks.cephadm:Deploying osd.0 on smithi026 with /dev/vg_nvme/lv_4... 2024-04-30T09:26:52.757 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-04-30T09:26:52.972 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:52 smithi046 systemd[1]: Starting Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:26:53.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:52 smithi046 podman[24356]: 2024-04-30 09:26:52.97128186 +0000 UTC m=+0.108221961 container create 2b8d2d89eb233f694b8cbd22c8e1a1e3be911145c0a81ff44a3ddcbb24d47a4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vcs-type=git, ceph=True, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, io.openshift.tags=base centos centos-stream, version=8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, architecture=x86_64, com.redhat.component=centos-stream-container, release=754) 2024-04-30T09:26:53.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:52 smithi046 podman[24356]: 2024-04-30 09:26:52.881905663 +0000 UTC m=+0.018845715 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a 2024-04-30T09:26:53.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:53 smithi046 podman[24356]: 2024-04-30 09:26:53.215441413 +0000 UTC m=+0.352381456 container init 2b8d2d89eb233f694b8cbd22c8e1a1e3be911145c0a81ff44a3ddcbb24d47a4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , name=centos-stream, architecture=x86_64, version=8, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image) 2024-04-30T09:26:53.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:53 smithi046 podman[24356]: 2024-04-30 09:26:53.21830922 +0000 UTC m=+0.355249263 container start 2b8d2d89eb233f694b8cbd22c8e1a1e3be911145c0a81ff44a3ddcbb24d47a4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, name=centos-stream, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, release=754, io.k8s.display-name=CentOS Stream 8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, version=8, io.buildah.version=1.19.8, vendor=Red Hat, Inc., RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-30T09:26:53.504 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-mgr[24375]: -- 172.21.15.46:0/313919572 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5621271b4340 con 0x56212718c800 2024-04-30T09:26:53.504 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:53 smithi046 bash[24356]: 2b8d2d89eb233f694b8cbd22c8e1a1e3be911145c0a81ff44a3ddcbb24d47a4e 2024-04-30T09:26:53.505 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:53 smithi046 systemd[1]: Started Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:26:53.505 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:53.365+0000 7f7187771000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-30T09:26:53.505 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:53.502+0000 7f7187771000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:26:53.937 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:53.611+0000 7f7187771000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-30T09:26:53.937 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-mon[23181]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:53.937 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:53.937 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:53.938 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:53.938 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[27123]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[22362]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:53.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:53 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:54.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:53.945+0000 7f7187771000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-30T09:26:54.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:54 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:54.033+0000 7f7187771000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-30T09:26:55.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:55.232+0000 7f7187771000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-30T09:26:55.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:55.402+0000 7f7187771000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-30T09:26:55.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:55.487+0000 7f7187771000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-30T09:26:55.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:55.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:55.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-mon[23181]: Reconfiguring mgr.y (unknown last config time)... 2024-04-30T09:26:55.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:26:55.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:26:55.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:55.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-mon[23181]: Reconfiguring daemon mgr.y on smithi026 2024-04-30T09:26:55.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-mon[23181]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[22362]: Reconfiguring mgr.y (unknown last config time)... 2024-04-30T09:26:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:26:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:26:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[22362]: Reconfiguring daemon mgr.y on smithi026 2024-04-30T09:26:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[22362]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:55.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:55.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:55.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[27123]: Reconfiguring mgr.y (unknown last config time)... 2024-04-30T09:26:55.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:26:55.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:26:55.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:55.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[27123]: Reconfiguring daemon mgr.y on smithi026 2024-04-30T09:26:55.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:55 smithi026 ceph-mon[27123]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:55.782 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:26:55.805 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_4 2024-04-30T09:26:56.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:55.808+0000 7f7187771000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-30T09:26:56.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:55.902+0000 7f7187771000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-30T09:26:56.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:55.997+0000 7f7187771000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-30T09:26:56.567 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:56 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:56.245+0000 7f7187771000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:26:56.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:26:56.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:56.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:26:56.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:26:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:26:56.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:56 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:56.702+0000 7f7187771000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-30T09:26:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:26:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:26:57.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:57.203+0000 7f7187771000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-30T09:26:57.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:57.299+0000 7f7187771000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-30T09:26:57.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:57.604+0000 7f7187771000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-30T09:26:57.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:57.695+0000 7f7187771000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-30T09:26:58.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[22362]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:58.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:58.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:58.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:58.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:58.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:58.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[27123]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:58.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:58.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:58.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:58.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:58.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:58 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:58.459 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-mon[23181]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:26:58.459 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:58.459 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:26:58.459 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:58.459 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:26:58.459 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:26:58.460 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:58.109+0000 7f7187771000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-30T09:26:58.460 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:58.245+0000 7f7187771000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-30T09:26:58.460 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:58.340+0000 7f7187771000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-30T09:26:58.460 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:58.457+0000 7f7187771000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-30T09:26:58.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:58.549+0000 7f7187771000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-30T09:26:59.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:59 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:26:59.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:59 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:26:59.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:26:59 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:59.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:59 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:26:59.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:59 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:26:59.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:26:59 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:59.483 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:59.115+0000 7f7187771000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-30T09:26:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:59 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:26:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:59 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:26:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:26:59 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:26:59.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:26:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:26:59.485+0000 7f7187771000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-30T09:27:00.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[22362]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:00.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[22362]: from='client.14223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:27:00.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[22362]: Standby manager daemon x started 2024-04-30T09:27:00.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[27123]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[27123]: from='client.14223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[27123]: Standby manager daemon x started 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:27:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:00 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:27:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:00 smithi046 ceph-mon[23181]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:00 smithi046 ceph-mon[23181]: from='client.14223 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:27:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:00 smithi046 ceph-mon[23181]: Standby manager daemon x started 2024-04-30T09:27:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:00 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:27:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:00 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:27:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:00 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:27:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:00 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/22134326' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:27:01.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:01 smithi026 ceph-mon[22362]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-30T09:27:01.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:01 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:27:01.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:01 smithi026 ceph-mon[27123]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-30T09:27:01.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:01 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:27:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:01 smithi046 ceph-mon[23181]: mgrmap e14: y(active, since 64s), standbys: x 2024-04-30T09:27:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:01 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[22362]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/574664129' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "031c2da1-c3a9-4d2b-83fe-93835cc5295a"}]: dispatch 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "031c2da1-c3a9-4d2b-83fe-93835cc5295a"}]: dispatch 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "031c2da1-c3a9-4d2b-83fe-93835cc5295a"}]': finished 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[22362]: osdmap e5: 1 total, 0 up, 1 in 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1787115605' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[27123]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/574664129' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "031c2da1-c3a9-4d2b-83fe-93835cc5295a"}]: dispatch 2024-04-30T09:27:02.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "031c2da1-c3a9-4d2b-83fe-93835cc5295a"}]: dispatch 2024-04-30T09:27:02.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "031c2da1-c3a9-4d2b-83fe-93835cc5295a"}]': finished 2024-04-30T09:27:02.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[27123]: osdmap e5: 1 total, 0 up, 1 in 2024-04-30T09:27:02.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:02.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:02 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1787115605' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:27:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:02 smithi046 ceph-mon[23181]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:02 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/574664129' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "031c2da1-c3a9-4d2b-83fe-93835cc5295a"}]: dispatch 2024-04-30T09:27:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:02 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "031c2da1-c3a9-4d2b-83fe-93835cc5295a"}]: dispatch 2024-04-30T09:27:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:02 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "031c2da1-c3a9-4d2b-83fe-93835cc5295a"}]': finished 2024-04-30T09:27:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:02 smithi046 ceph-mon[23181]: osdmap e5: 1 total, 0 up, 1 in 2024-04-30T09:27:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:02 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:02 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1787115605' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:27:04.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:04 smithi026 ceph-mon[22362]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:04.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:04 smithi026 ceph-mon[27123]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:04.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:04 smithi046 ceph-mon[23181]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:06.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:06 smithi026 ceph-mon[22362]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:06.222 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:06 smithi026 ceph-mon[27123]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:06 smithi046 ceph-mon[23181]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:08.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:08 smithi026 ceph-mon[27123]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:08.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:08 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-30T09:27:08.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:08 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:08.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:08 smithi026 ceph-mon[22362]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:08.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:08 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-30T09:27:08.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:08 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:08.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:08 smithi046 ceph-mon[23181]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:08.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:08 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-30T09:27:08.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:08 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:09.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:09 smithi026 ceph-mon[22362]: Deploying daemon osd.0 on smithi026 2024-04-30T09:27:09.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:09 smithi026 ceph-mon[27123]: Deploying daemon osd.0 on smithi026 2024-04-30T09:27:09.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:09 smithi046 ceph-mon[23181]: Deploying daemon osd.0 on smithi026 2024-04-30T09:27:10.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:10 smithi026 ceph-mon[27123]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:10.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:10 smithi026 ceph-mon[22362]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:10.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:10 smithi046 ceph-mon[23181]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:11.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:11 smithi046 ceph-mon[23181]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:11.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:11 smithi026 ceph-mon[22362]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:11.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:11 smithi026 ceph-mon[27123]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:14.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[27123]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:14.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:14.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:14.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:14.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:14.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[22362]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:14.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:14.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:14.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:14.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:13 smithi046 ceph-mon[23181]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:13 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:13 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:13 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:13 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:15.625 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 0 on host 'smithi026' 2024-04-30T09:27:15.878 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:15 smithi026 ceph-mon[22362]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:15.878 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:15 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:15.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:15 smithi026 ceph-mon[27123]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:15.878 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:15 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:16.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:15 smithi046 ceph-mon[23181]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:16.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:15 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:16.312 DEBUG:teuthology.orchestra.run.smithi026:osd.0> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.0.service 2024-04-30T09:27:16.314 INFO:tasks.cephadm:Deploying osd.1 on smithi026 with /dev/vg_nvme/lv_3... 2024-04-30T09:27:16.315 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-04-30T09:27:16.999 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:16 smithi026 ceph-mon[22362]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-30T09:27:17.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:16 smithi026 ceph-mon[27123]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-30T09:27:17.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:16 smithi046 ceph-mon[23181]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-30T09:27:18.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[27123]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:18.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[27123]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-30T09:27:18.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[27123]: osdmap e6: 1 total, 0 up, 1 in 2024-04-30T09:27:18.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:18.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[27123]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:27:18.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:18.071 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[30006]: 2024-04-30T09:27:17.834+0000 7f6d9cb48700 -1 osd.0 0 waiting for initial osdmap 2024-04-30T09:27:18.072 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[30006]: 2024-04-30T09:27:17.838+0000 7f6d95cdc700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:27:18.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[22362]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:18.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[22362]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-30T09:27:18.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[22362]: osdmap e6: 1 total, 0 up, 1 in 2024-04-30T09:27:18.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:18.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[22362]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:27:18.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:17 smithi046 ceph-mon[23181]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:17 smithi046 ceph-mon[23181]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-30T09:27:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:17 smithi046 ceph-mon[23181]: osdmap e6: 1 total, 0 up, 1 in 2024-04-30T09:27:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:17 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:17 smithi046 ceph-mon[23181]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:27:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:17 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:19.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:18 smithi026 ceph-mon[27123]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:27:19.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:18 smithi026 ceph-mon[27123]: osdmap e7: 1 total, 0 up, 1 in 2024-04-30T09:27:19.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:18 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:18 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:18 smithi026 ceph-mon[22362]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:27:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:18 smithi026 ceph-mon[22362]: osdmap e7: 1 total, 0 up, 1 in 2024-04-30T09:27:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:18 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:18 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:19.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:18 smithi046 ceph-mon[23181]: from='osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:27:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:18 smithi046 ceph-mon[23181]: osdmap e7: 1 total, 0 up, 1 in 2024-04-30T09:27:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:18 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:18 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:19.599 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:27:19.624 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_3 2024-04-30T09:27:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[22362]: purged_snaps scrub starts 2024-04-30T09:27:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[22362]: purged_snaps scrub ok 2024-04-30T09:27:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[22362]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[22362]: osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558] boot 2024-04-30T09:27:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[22362]: osdmap e8: 1 total, 1 up, 1 in 2024-04-30T09:27:19.882 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[27123]: purged_snaps scrub starts 2024-04-30T09:27:19.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[27123]: purged_snaps scrub ok 2024-04-30T09:27:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[27123]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[27123]: osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558] boot 2024-04-30T09:27:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[27123]: osdmap e8: 1 total, 1 up, 1 in 2024-04-30T09:27:19.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:19 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:20.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:19 smithi046 ceph-mon[23181]: purged_snaps scrub starts 2024-04-30T09:27:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:19 smithi046 ceph-mon[23181]: purged_snaps scrub ok 2024-04-30T09:27:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:19 smithi046 ceph-mon[23181]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-30T09:27:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:19 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:19 smithi046 ceph-mon[23181]: osd.0 [v2:172.21.15.26:6802/2102761558,v1:172.21.15.26:6803/2102761558] boot 2024-04-30T09:27:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:19 smithi046 ceph-mon[23181]: osdmap e8: 1 total, 1 up, 1 in 2024-04-30T09:27:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:19 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:27:21.099 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:20 smithi026 ceph-mon[22362]: osdmap e9: 1 total, 1 up, 1 in 2024-04-30T09:27:21.099 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:20 smithi026 ceph-mon[27123]: osdmap e9: 1 total, 1 up, 1 in 2024-04-30T09:27:21.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:20 smithi046 ceph-mon[23181]: osdmap e9: 1 total, 1 up, 1 in 2024-04-30T09:27:22.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:21 smithi026 ceph-mon[22362]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:22.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:21 smithi026 ceph-mon[27123]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:22.204 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:21 smithi046 ceph-mon[23181]: pgmap v41: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:23.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: Detected new or changed devices on smithi026 2024-04-30T09:27:23.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi026 to 16069M 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi026 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:27:23.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi026 to 16069M 2024-04-30T09:27:23.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:23.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:23.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:23.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: Detected new or changed devices on smithi026 2024-04-30T09:27:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:27:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi026 to 16069M 2024-04-30T09:27:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:24.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[22362]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:24.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[22362]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:27:24.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:27:24.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:27:24.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:24.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[27123]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:24.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[27123]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:27:24.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:27:24.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:27:24.061 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:23 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:24.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:23 smithi046 ceph-mon[23181]: pgmap v42: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:23 smithi046 ceph-mon[23181]: from='client.14250 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:27:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:23 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:27:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:23 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:27:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:23 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[22362]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1399629925' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59928493-d00b-4259-b638-8b4743f1434e"}]: dispatch 2024-04-30T09:27:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1399629925' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59928493-d00b-4259-b638-8b4743f1434e"}]': finished 2024-04-30T09:27:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[22362]: osdmap e10: 2 total, 1 up, 2 in 2024-04-30T09:27:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:26.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[27123]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:26.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1399629925' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59928493-d00b-4259-b638-8b4743f1434e"}]: dispatch 2024-04-30T09:27:26.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1399629925' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59928493-d00b-4259-b638-8b4743f1434e"}]': finished 2024-04-30T09:27:26.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[27123]: osdmap e10: 2 total, 1 up, 2 in 2024-04-30T09:27:26.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:25 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:26.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:25 smithi046 ceph-mon[23181]: pgmap v43: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:25 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1399629925' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "59928493-d00b-4259-b638-8b4743f1434e"}]: dispatch 2024-04-30T09:27:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:25 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1399629925' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "59928493-d00b-4259-b638-8b4743f1434e"}]': finished 2024-04-30T09:27:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:25 smithi046 ceph-mon[23181]: osdmap e10: 2 total, 1 up, 2 in 2024-04-30T09:27:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:25 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:27.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:26 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2005788390' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:27:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:26 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2005788390' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:27:27.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:26 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2005788390' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:27:28.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:27 smithi026 ceph-mon[22362]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:27 smithi026 ceph-mon[27123]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:28.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:27 smithi046 ceph-mon[23181]: pgmap v45: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:29.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:29 smithi026 ceph-mon[27123]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:29.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:29 smithi026 ceph-mon[22362]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:29 smithi046 ceph-mon[23181]: pgmap v46: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:31.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:31 smithi026 ceph-mon[22362]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:31.591 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:31 smithi026 ceph-mon[27123]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:31 smithi046 ceph-mon[23181]: pgmap v47: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:32.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-30T09:27:32.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:32.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:32 smithi026 ceph-mon[22362]: Deploying daemon osd.1 on smithi026 2024-04-30T09:27:32.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:32 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-30T09:27:32.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:32 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:32.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:32 smithi026 ceph-mon[27123]: Deploying daemon osd.1 on smithi026 2024-04-30T09:27:32.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:32 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-30T09:27:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:32 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:32 smithi046 ceph-mon[23181]: Deploying daemon osd.1 on smithi026 2024-04-30T09:27:33.510 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:33 smithi026 ceph-mon[27123]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:33.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:33 smithi026 ceph-mon[22362]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:33.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:33 smithi046 ceph-mon[23181]: pgmap v48: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:35.940 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:35 smithi026 ceph-mon[22362]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:35.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:35 smithi026 ceph-mon[27123]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:35.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:35 smithi046 ceph-mon[23181]: pgmap v49: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:38.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[27123]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:38.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:38.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:38.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:38.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:38.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[22362]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:38.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:38.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:38.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:38.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:37 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:37 smithi046 ceph-mon[23181]: pgmap v50: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:39.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:39 smithi026 ceph-mon[22362]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:39.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:39 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:39.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:39 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:39.908 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:39 smithi026 ceph-mon[27123]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:39.908 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:39 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:39.908 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:39 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:39 smithi046 ceph-mon[23181]: pgmap v51: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:39 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:39 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:40.356 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 1 on host 'smithi026' 2024-04-30T09:27:40.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[22362]: from='osd.1 [v2:172.21.15.26:6810/1899832674,v1:172.21.15.26:6811/1899832674]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:27:40.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[22362]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[27123]: from='osd.1 [v2:172.21.15.26:6810/1899832674,v1:172.21.15.26:6811/1899832674]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[27123]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:40.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:40 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:41.008 DEBUG:teuthology.orchestra.run.smithi026:osd.1> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.1.service 2024-04-30T09:27:41.010 INFO:tasks.cephadm:Deploying osd.2 on smithi026 with /dev/vg_nvme/lv_2... 2024-04-30T09:27:41.011 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-04-30T09:27:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:40 smithi046 ceph-mon[23181]: from='osd.1 [v2:172.21.15.26:6810/1899832674,v1:172.21.15.26:6811/1899832674]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:27:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:40 smithi046 ceph-mon[23181]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:27:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:40 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:40 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:27:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:40 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:40 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:27:41.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[22362]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:41.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[22362]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-30T09:27:41.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[22362]: osdmap e11: 2 total, 1 up, 2 in 2024-04-30T09:27:41.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:41.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[22362]: from='osd.1 [v2:172.21.15.26:6810/1899832674,v1:172.21.15.26:6811/1899832674]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:27:41.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[22362]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:27:41.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[27123]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:41.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[27123]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-30T09:27:41.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[27123]: osdmap e11: 2 total, 1 up, 2 in 2024-04-30T09:27:41.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:41.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[27123]: from='osd.1 [v2:172.21.15.26:6810/1899832674,v1:172.21.15.26:6811/1899832674]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:27:41.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-mon[27123]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:27:41.891 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[32991]: 2024-04-30T09:27:41.798+0000 7fce75146700 -1 osd.1 0 waiting for initial osdmap 2024-04-30T09:27:41.891 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:27:41 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[32991]: 2024-04-30T09:27:41.801+0000 7fce6c2d6700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:27:42.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:41 smithi046 ceph-mon[23181]: pgmap v52: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:41 smithi046 ceph-mon[23181]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-30T09:27:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:41 smithi046 ceph-mon[23181]: osdmap e11: 2 total, 1 up, 2 in 2024-04-30T09:27:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:41 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:41 smithi046 ceph-mon[23181]: from='osd.1 [v2:172.21.15.26:6810/1899832674,v1:172.21.15.26:6811/1899832674]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:27:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:41 smithi046 ceph-mon[23181]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:27:43.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:42 smithi026 ceph-mon[22362]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:27:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:42 smithi026 ceph-mon[22362]: osdmap e12: 2 total, 1 up, 2 in 2024-04-30T09:27:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:42 smithi026 ceph-mon[27123]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:27:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:42 smithi026 ceph-mon[27123]: osdmap e12: 2 total, 1 up, 2 in 2024-04-30T09:27:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:42 smithi046 ceph-mon[23181]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:27:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:42 smithi046 ceph-mon[23181]: osdmap e12: 2 total, 1 up, 2 in 2024-04-30T09:27:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:42 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:42 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[22362]: purged_snaps scrub starts 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[22362]: purged_snaps scrub ok 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[22362]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[22362]: osd.1 [v2:172.21.15.26:6810/1899832674,v1:172.21.15.26:6811/1899832674] boot 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[22362]: osdmap e13: 2 total, 2 up, 2 in 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[27123]: purged_snaps scrub starts 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[27123]: purged_snaps scrub ok 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[27123]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[27123]: osd.1 [v2:172.21.15.26:6810/1899832674,v1:172.21.15.26:6811/1899832674] boot 2024-04-30T09:27:44.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[27123]: osdmap e13: 2 total, 2 up, 2 in 2024-04-30T09:27:44.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:43 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:43 smithi046 ceph-mon[23181]: purged_snaps scrub starts 2024-04-30T09:27:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:43 smithi046 ceph-mon[23181]: purged_snaps scrub ok 2024-04-30T09:27:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:43 smithi046 ceph-mon[23181]: pgmap v55: 0 pgs: ; 0 B data, 4.8 MiB used, 89 GiB / 89 GiB avail 2024-04-30T09:27:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:43 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:43 smithi046 ceph-mon[23181]: osd.1 [v2:172.21.15.26:6810/1899832674,v1:172.21.15.26:6811/1899832674] boot 2024-04-30T09:27:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:43 smithi046 ceph-mon[23181]: osdmap e13: 2 total, 2 up, 2 in 2024-04-30T09:27:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:43 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:27:44.920 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[22362]: osdmap e14: 2 total, 2 up, 2 in 2024-04-30T09:27:44.920 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:44.920 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:27:44.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:27:44.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:44.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:44.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[27123]: osdmap e14: 2 total, 2 up, 2 in 2024-04-30T09:27:44.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:44.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:27:44.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:27:44.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:44.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:44 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:45.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:44 smithi046 ceph-mon[23181]: osdmap e14: 2 total, 2 up, 2 in 2024-04-30T09:27:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:27:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:27:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:27:46.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:45 smithi026 ceph-mon[22362]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:46.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:45 smithi026 ceph-mon[22362]: Detected new or changed devices on smithi026 2024-04-30T09:27:46.005 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:45 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi026 to 8034M 2024-04-30T09:27:46.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:45 smithi026 ceph-mon[27123]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:46.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:45 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi026 2024-04-30T09:27:46.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:45 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi026 to 8034M 2024-04-30T09:27:46.010 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:27:46.031 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_2 2024-04-30T09:27:46.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:45 smithi046 ceph-mon[23181]: pgmap v58: 0 pgs: ; 0 B data, 9.7 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:45 smithi046 ceph-mon[23181]: Detected new or changed devices on smithi026 2024-04-30T09:27:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:45 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi026 to 8034M 2024-04-30T09:27:47.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:47 smithi026 ceph-mon[22362]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:47 smithi026 ceph-mon[27123]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:48.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:47 smithi046 ceph-mon[23181]: pgmap v59: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:49.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:48 smithi026 ceph-mon[22362]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:27:49.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:27:49.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:27:49.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:49.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:48 smithi026 ceph-mon[27123]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:27:49.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:27:49.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:27:49.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:49.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:48 smithi046 ceph-mon[23181]: from='client.24178 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:27:49.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:27:49.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:27:49.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:49.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:49 smithi026 ceph-mon[27123]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:49.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:49 smithi026 ceph-mon[22362]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:50.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:49 smithi046 ceph-mon[23181]: pgmap v60: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:51.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:50 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/325910941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "07cad5bf-069c-4688-8de5-abcf2b56f970"}]: dispatch 2024-04-30T09:27:51.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:50 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/325910941' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "07cad5bf-069c-4688-8de5-abcf2b56f970"}]': finished 2024-04-30T09:27:51.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:50 smithi026 ceph-mon[22362]: osdmap e15: 3 total, 2 up, 3 in 2024-04-30T09:27:51.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:50 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:27:51.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:50 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/325910941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "07cad5bf-069c-4688-8de5-abcf2b56f970"}]: dispatch 2024-04-30T09:27:51.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:50 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/325910941' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "07cad5bf-069c-4688-8de5-abcf2b56f970"}]': finished 2024-04-30T09:27:51.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:50 smithi026 ceph-mon[27123]: osdmap e15: 3 total, 2 up, 3 in 2024-04-30T09:27:51.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:50 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:27:51.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:50 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/325910941' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "07cad5bf-069c-4688-8de5-abcf2b56f970"}]: dispatch 2024-04-30T09:27:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:50 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/325910941' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "07cad5bf-069c-4688-8de5-abcf2b56f970"}]': finished 2024-04-30T09:27:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:50 smithi046 ceph-mon[23181]: osdmap e15: 3 total, 2 up, 3 in 2024-04-30T09:27:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:50 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:27:52.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:51 smithi026 ceph-mon[22362]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:52.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:51 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2932596407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:27:52.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:51 smithi026 ceph-mon[27123]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:52.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:51 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2932596407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:27:52.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:51 smithi046 ceph-mon[23181]: pgmap v62: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:51 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2932596407' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:27:54.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:53 smithi026 ceph-mon[22362]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:54.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:53 smithi026 ceph-mon[27123]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:53 smithi046 ceph-mon[23181]: pgmap v63: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:55.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:55 smithi026 ceph-mon[22362]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:55.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:55 smithi026 ceph-mon[27123]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:55 smithi046 ceph-mon[23181]: pgmap v64: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:57.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:27:57.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:27:57.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-30T09:27:57.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:57.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:27:57.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:27:57.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-30T09:27:57.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:57.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:27:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:27:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-30T09:27:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:27:58.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:57 smithi026 ceph-mon[22362]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:58.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:57 smithi026 ceph-mon[22362]: Deploying daemon osd.2 on smithi026 2024-04-30T09:27:58.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:57 smithi026 ceph-mon[27123]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:58.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:57 smithi026 ceph-mon[27123]: Deploying daemon osd.2 on smithi026 2024-04-30T09:27:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:57 smithi046 ceph-mon[23181]: pgmap v65: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:27:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:57 smithi046 ceph-mon[23181]: Deploying daemon osd.2 on smithi026 2024-04-30T09:28:00.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:27:59 smithi026 ceph-mon[22362]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:00.161 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:27:59 smithi026 ceph-mon[27123]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:00.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:27:59 smithi046 ceph-mon[23181]: pgmap v66: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:01.902 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:01 smithi026 ceph-mon[22362]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:01.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:01 smithi026 ceph-mon[27123]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:02.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:01 smithi046 ceph-mon[23181]: pgmap v67: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:03.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:02 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:03.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:02 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:03.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:02 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:03.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:02 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:03.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:02 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:03.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:02 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:03.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:02 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:03.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:02 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:02 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:02 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:02 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:02 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:03 smithi026 ceph-mon[27123]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:03 smithi026 ceph-mon[22362]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:04.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:03 smithi046 ceph-mon[23181]: pgmap v68: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:04.635 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 2 on host 'smithi026' 2024-04-30T09:28:05.567 DEBUG:teuthology.orchestra.run.smithi026:osd.2> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.2.service 2024-04-30T09:28:05.570 INFO:tasks.cephadm:Deploying osd.3 on smithi026 with /dev/vg_nvme/lv_1... 2024-04-30T09:28:05.570 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-04-30T09:28:05.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:05 smithi026 ceph-mon[27123]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:05.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:05 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:05.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:05 smithi026 ceph-mon[27123]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-30T09:28:05.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:05 smithi026 ceph-mon[22362]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:05.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:05 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:05.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:05 smithi026 ceph-mon[22362]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-30T09:28:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:05 smithi046 ceph-mon[23181]: pgmap v69: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:05 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:05 smithi046 ceph-mon[23181]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-30T09:28:07.046 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:07.046 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[27123]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-30T09:28:07.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[27123]: osdmap e16: 3 total, 2 up, 3 in 2024-04-30T09:28:07.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:07.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[27123]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:28:07.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:07.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[22362]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-30T09:28:07.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[22362]: osdmap e16: 3 total, 2 up, 3 in 2024-04-30T09:28:07.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:07.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-mon[22362]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:28:07.047 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[35911]: 2024-04-30T09:28:06.893+0000 7fa652052700 -1 osd.2 0 waiting for initial osdmap 2024-04-30T09:28:07.048 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:28:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[35911]: 2024-04-30T09:28:06.897+0000 7fa6499e3700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:28:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:06 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:06 smithi046 ceph-mon[23181]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-30T09:28:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:06 smithi046 ceph-mon[23181]: osdmap e16: 3 total, 2 up, 3 in 2024-04-30T09:28:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:06 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:06 smithi046 ceph-mon[23181]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:28:08.046 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[27123]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:08.046 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[27123]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:28:08.046 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[27123]: osdmap e17: 3 total, 2 up, 3 in 2024-04-30T09:28:08.046 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:08.046 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:08.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[22362]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:08.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[22362]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:28:08.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[22362]: osdmap e17: 3 total, 2 up, 3 in 2024-04-30T09:28:08.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:08.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:07 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:08.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:07 smithi046 ceph-mon[23181]: pgmap v71: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:07 smithi046 ceph-mon[23181]: from='osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:28:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:07 smithi046 ceph-mon[23181]: osdmap e17: 3 total, 2 up, 3 in 2024-04-30T09:28:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:07 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:07 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:08.660 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:28:08.679 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi026:vg_nvme/lv_1 2024-04-30T09:28:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:08 smithi046 ceph-mon[23181]: purged_snaps scrub starts 2024-04-30T09:28:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:08 smithi046 ceph-mon[23181]: purged_snaps scrub ok 2024-04-30T09:28:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:08 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:08 smithi046 ceph-mon[23181]: osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132] boot 2024-04-30T09:28:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:08 smithi046 ceph-mon[23181]: osdmap e18: 3 total, 3 up, 3 in 2024-04-30T09:28:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:08 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:08 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-30T09:28:09.247 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[27123]: purged_snaps scrub starts 2024-04-30T09:28:09.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[27123]: purged_snaps scrub ok 2024-04-30T09:28:09.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:09.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[27123]: osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132] boot 2024-04-30T09:28:09.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[27123]: osdmap e18: 3 total, 3 up, 3 in 2024-04-30T09:28:09.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:09.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-30T09:28:09.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[22362]: purged_snaps scrub starts 2024-04-30T09:28:09.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[22362]: purged_snaps scrub ok 2024-04-30T09:28:09.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:09.249 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[22362]: osd.2 [v2:172.21.15.26:6818/3754664132,v1:172.21.15.26:6819/3754664132] boot 2024-04-30T09:28:09.249 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[22362]: osdmap e18: 3 total, 3 up, 3 in 2024-04-30T09:28:09.249 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:28:09.249 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:08 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-04-30T09:28:10.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:09 smithi026 ceph-mon[27123]: pgmap v74: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:10.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-30T09:28:10.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:09 smithi026 ceph-mon[27123]: osdmap e19: 3 total, 3 up, 3 in 2024-04-30T09:28:10.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-30T09:28:10.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:09 smithi026 ceph-mon[22362]: pgmap v74: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:10.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-30T09:28:10.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:09 smithi026 ceph-mon[22362]: osdmap e19: 3 total, 3 up, 3 in 2024-04-30T09:28:10.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-30T09:28:10.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:09 smithi046 ceph-mon[23181]: pgmap v74: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2024-04-30T09:28:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-04-30T09:28:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:09 smithi046 ceph-mon[23181]: osdmap e19: 3 total, 3 up, 3 in 2024-04-30T09:28:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-30T09:28:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:10 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-30T09:28:11.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:10 smithi026 ceph-mon[27123]: osdmap e20: 3 total, 3 up, 3 in 2024-04-30T09:28:11.417 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:28:11 smithi026 sudo[37223]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-30T09:28:11.417 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:28:11 smithi026 sudo[37223]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:11.417 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:28:11 smithi026 sudo[37223]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:11.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:10 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-30T09:28:11.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:10 smithi026 ceph-mon[22362]: osdmap e20: 3 total, 3 up, 3 in 2024-04-30T09:28:11.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:10 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-30T09:28:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:10 smithi046 ceph-mon[23181]: osdmap e20: 3 total, 3 up, 3 in 2024-04-30T09:28:11.696 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:28:11 smithi026 sudo[37244]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-30T09:28:11.696 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:28:11 smithi026 sudo[37244]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:11.696 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:28:11 smithi026 sudo[37244]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:12.446 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:28:12 smithi026 sudo[37273]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-30T09:28:12.446 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:28:12 smithi026 sudo[37273]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:12.446 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:28:12 smithi026 sudo[37273]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: Detected new or changed devices on smithi026 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi026 to 5356M 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: osdmap e21: 3 total, 3 up, 3 in 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi026 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi026 to 5356M 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:12.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:12.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:12.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: osdmap e21: 3 total, 3 up, 3 in 2024-04-30T09:28:12.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:12 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: pgmap v77: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: Detected new or changed devices on smithi026 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi026 to 5356M 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: osdmap e21: 3 total, 3 up, 3 in 2024-04-30T09:28:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:12 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:12.769 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:28:12 smithi026 sudo[37353]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-30T09:28:12.769 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:28:12 smithi026 sudo[37353]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:12.769 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:28:12 smithi026 sudo[37353]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:13.019 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:28:13 smithi026 sudo[37375]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-30T09:28:13.019 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:28:13 smithi026 sudo[37375]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:13.019 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[22362]: from='client.14310 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:28:13.020 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:28:13.020 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:28:13.020 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:13.020 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[22362]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-30T09:28:13.446 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:28:13 smithi026 sudo[37375]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[27123]: from='client.14310 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:28:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:28:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:28:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:13 smithi026 ceph-mon[27123]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-30T09:28:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:13 smithi046 ceph-mon[23181]: from='client.14310 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:28:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:13 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:28:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:13 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:28:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:13 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:13 smithi046 ceph-mon[23181]: mgrmap e15: y(active, since 2m), standbys: x 2024-04-30T09:28:13.820 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:28:13 smithi026 sudo[37382]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-30T09:28:13.820 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:28:13 smithi026 sudo[37382]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:13.820 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:28:13 smithi026 sudo[37382]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:14.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:13 smithi026 sudo[37408]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-30T09:28:14.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:13 smithi026 sudo[37408]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:14.081 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:13 smithi026 sudo[37408]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:14.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:14 smithi026 ceph-mon[27123]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:14.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:14 smithi026 sudo[37426]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-30T09:28:14.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:14 smithi026 sudo[37426]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:14.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:14 smithi026 sudo[37426]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:14 smithi026 ceph-mon[22362]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:14.733 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:14 smithi046 ceph-mon[23181]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:15.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:14 smithi026 sudo[37450]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-30T09:28:15.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:14 smithi026 sudo[37450]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:15.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:14 smithi026 sudo[37450]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:15.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:15 smithi026 sudo[37453]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-30T09:28:15.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:15 smithi026 sudo[37453]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:15.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:15 smithi026 sudo[37453]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:15.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:15 smithi046 sudo[24696]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2024-04-30T09:28:15.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:15 smithi046 sudo[24696]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:15.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:15 smithi046 sudo[24696]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:16.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1102489934' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5e016838-5979-4375-aae0-8a42a81a5290"}]: dispatch 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1102489934' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5e016838-5979-4375-aae0-8a42a81a5290"}]': finished 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: osdmap e22: 4 total, 3 up, 4 in 2024-04-30T09:28:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:16.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:28:16.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:28:16.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:28:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-30T09:28:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-30T09:28:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:28:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:28:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:28:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1102489934' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5e016838-5979-4375-aae0-8a42a81a5290"}]: dispatch 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1102489934' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5e016838-5979-4375-aae0-8a42a81a5290"}]': finished 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: osdmap e22: 4 total, 3 up, 4 in 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:28:16.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:16 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:28:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 sudo[24699]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-04-30T09:28:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 sudo[24699]: pam_unix(sudo:session): session opened for user root by (uid=0) 2024-04-30T09:28:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 sudo[24699]: pam_unix(sudo:session): session closed for user root 2024-04-30T09:28:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-30T09:28:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-30T09:28:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:28:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:28:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:28:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1102489934' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5e016838-5979-4375-aae0-8a42a81a5290"}]: dispatch 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1102489934' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5e016838-5979-4375-aae0-8a42a81a5290"}]': finished 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: osdmap e22: 4 total, 3 up, 4 in 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:28:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:16 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:28:17.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:17 smithi026 ceph-mon[22362]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-30T09:28:17.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:17 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/541141821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:28:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:17 smithi026 ceph-mon[22362]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-30T09:28:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:17 smithi026 ceph-mon[27123]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-30T09:28:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:17 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/541141821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:28:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:17 smithi026 ceph-mon[27123]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-30T09:28:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:17 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:17.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:17 smithi046 ceph-mon[23181]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-30T09:28:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:17 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/541141821' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:28:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:17 smithi046 ceph-mon[23181]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-30T09:28:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:17 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:18.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:18 smithi046 ceph-mon[23181]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:18.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:18 smithi026 ceph-mon[22362]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:18.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:18 smithi026 ceph-mon[27123]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:20 smithi046 ceph-mon[23181]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:20.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:20 smithi026 ceph-mon[22362]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:20.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:20 smithi026 ceph-mon[27123]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:22 smithi026 ceph-mon[22362]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-30T09:28:22.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:22 smithi026 ceph-mon[27123]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-30T09:28:22.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-30T09:28:22.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:22 smithi046 ceph-mon[23181]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-30T09:28:22.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-30T09:28:22.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:22 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:22.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:22 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:22.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-30T09:28:22.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:22 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:23.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:23 smithi026 ceph-mon[22362]: Deploying daemon osd.3 on smithi026 2024-04-30T09:28:23.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:23 smithi026 ceph-mon[27123]: Deploying daemon osd.3 on smithi026 2024-04-30T09:28:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:23 smithi046 ceph-mon[23181]: Deploying daemon osd.3 on smithi026 2024-04-30T09:28:24.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:24 smithi026 ceph-mon[27123]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-30T09:28:24.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:24 smithi026 ceph-mon[22362]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-30T09:28:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:24 smithi046 ceph-mon[23181]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-30T09:28:26.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:26 smithi026 ceph-mon[22362]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-30T09:28:26.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:26 smithi026 ceph-mon[27123]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-30T09:28:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:26 smithi046 ceph-mon[23181]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-30T09:28:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:28 smithi046 ceph-mon[23181]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 4 op/s 2024-04-30T09:28:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:28 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:28 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:28 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:28 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[27123]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 4 op/s 2024-04-30T09:28:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:28.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:28.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[22362]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 4 op/s 2024-04-30T09:28:28.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:28.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:28.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:28.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:28 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:30.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:30 smithi026 ceph-mon[22362]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-30T09:28:30.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:30 smithi026 ceph-mon[27123]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-30T09:28:30.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:30 smithi046 ceph-mon[23181]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-30T09:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 3 on host 'smithi026' 2024-04-30T09:28:31.181 DEBUG:teuthology.orchestra.run.smithi026:osd.3> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.3.service 2024-04-30T09:28:31.184 INFO:tasks.cephadm:Deploying osd.4 on smithi046 with /dev/vg_nvme/lv_4... 2024-04-30T09:28:31.184 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-04-30T09:28:31.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:31 smithi026 ceph-mon[22362]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-30T09:28:31.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:31 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:31.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:31 smithi026 ceph-mon[27123]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-30T09:28:31.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:31 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:31 smithi046 ceph-mon[23181]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-30T09:28:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:31 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:32.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[27123]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-30T09:28:32.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[27123]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-30T09:28:32.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[27123]: osdmap e23: 4 total, 3 up, 4 in 2024-04-30T09:28:32.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:32.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[27123]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:28:32.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:32.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[22362]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-30T09:28:32.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[22362]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-30T09:28:32.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[22362]: osdmap e23: 4 total, 3 up, 4 in 2024-04-30T09:28:32.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:32.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[22362]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:28:32.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:32.376 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[38920]: 2024-04-30T09:28:32.241+0000 7f27580b8700 -1 osd.3 0 waiting for initial osdmap 2024-04-30T09:28:32.376 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 09:28:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[38920]: 2024-04-30T09:28:32.245+0000 7f275224e700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:28:32.509 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:32 smithi046 ceph-mon[23181]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-30T09:28:32.509 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:32 smithi046 ceph-mon[23181]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-30T09:28:32.509 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:32 smithi046 ceph-mon[23181]: osdmap e23: 4 total, 3 up, 4 in 2024-04-30T09:28:32.509 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:32 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:32.509 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:32 smithi046 ceph-mon[23181]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:28:32.509 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:32 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:33.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:33 smithi026 ceph-mon[27123]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:28:33.309 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:33 smithi026 ceph-mon[27123]: osdmap e24: 4 total, 3 up, 4 in 2024-04-30T09:28:33.309 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:33 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:33.309 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:33 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:33.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:33 smithi026 ceph-mon[22362]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:28:33.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:33 smithi026 ceph-mon[22362]: osdmap e24: 4 total, 3 up, 4 in 2024-04-30T09:28:33.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:33 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:33.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:33 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:33.453 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:33 smithi046 ceph-mon[23181]: from='osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2024-04-30T09:28:33.453 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:33 smithi046 ceph-mon[23181]: osdmap e24: 4 total, 3 up, 4 in 2024-04-30T09:28:33.453 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:33 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:33.453 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:33 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:33.940 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:28:33.963 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi046:vg_nvme/lv_4 2024-04-30T09:28:34.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[27123]: purged_snaps scrub starts 2024-04-30T09:28:34.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[27123]: purged_snaps scrub ok 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[27123]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[27123]: osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924] boot 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[27123]: osdmap e25: 4 total, 4 up, 4 in 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[22362]: purged_snaps scrub starts 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[22362]: purged_snaps scrub ok 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[22362]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[22362]: osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924] boot 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[22362]: osdmap e25: 4 total, 4 up, 4 in 2024-04-30T09:28:34.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:34 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:34.580 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:34 smithi046 ceph-mon[23181]: purged_snaps scrub starts 2024-04-30T09:28:34.580 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:34 smithi046 ceph-mon[23181]: purged_snaps scrub ok 2024-04-30T09:28:34.580 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:34 smithi046 ceph-mon[23181]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2024-04-30T09:28:34.580 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:34 smithi046 ceph-mon[23181]: osd.3 [v2:172.21.15.26:6826/1106154924,v1:172.21.15.26:6827/1106154924] boot 2024-04-30T09:28:34.580 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:34 smithi046 ceph-mon[23181]: osdmap e25: 4 total, 4 up, 4 in 2024-04-30T09:28:34.580 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:34 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:28:35.459 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: osdmap e26: 4 total, 4 up, 4 in 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: Detected new or changed devices on smithi026 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi026 to 4017M 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:35.460 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:35.461 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:35 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: osdmap e26: 4 total, 4 up, 4 in 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: Detected new or changed devices on smithi026 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi026 to 4017M 2024-04-30T09:28:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: osdmap e26: 4 total, 4 up, 4 in 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi026 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:35.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi026 to 4017M 2024-04-30T09:28:35.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:35.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:35.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:35.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:35.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:35 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:36.672 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:36 smithi046 ceph-mon[23181]: from='client.24215 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:28:36.673 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:36 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:28:36.673 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:36 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:28:36.673 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:36 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:36.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:36 smithi026 ceph-mon[27123]: from='client.24215 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:28:36.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:36 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:28:36.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:36 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:28:36.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:36 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:36 smithi026 ceph-mon[22362]: from='client.24215 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:28:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:36 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:28:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:36 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:28:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:36 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:37.544 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:37 smithi046 ceph-mon[23181]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:37.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:37 smithi026 ceph-mon[27123]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:37.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:37 smithi026 ceph-mon[22362]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:38.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/1119250517' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4ea38ab-de25-4ee8-a55c-e75a516ff355"}]: dispatch 2024-04-30T09:28:38.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4ea38ab-de25-4ee8-a55c-e75a516ff355"}]: dispatch 2024-04-30T09:28:38.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c4ea38ab-de25-4ee8-a55c-e75a516ff355"}]': finished 2024-04-30T09:28:38.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[27123]: osdmap e27: 5 total, 4 up, 5 in 2024-04-30T09:28:38.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:38.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/1119250517' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4ea38ab-de25-4ee8-a55c-e75a516ff355"}]: dispatch 2024-04-30T09:28:38.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4ea38ab-de25-4ee8-a55c-e75a516ff355"}]: dispatch 2024-04-30T09:28:38.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c4ea38ab-de25-4ee8-a55c-e75a516ff355"}]': finished 2024-04-30T09:28:38.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[22362]: osdmap e27: 5 total, 4 up, 5 in 2024-04-30T09:28:38.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:38 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:38 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/1119250517' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4ea38ab-de25-4ee8-a55c-e75a516ff355"}]: dispatch 2024-04-30T09:28:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:38 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c4ea38ab-de25-4ee8-a55c-e75a516ff355"}]: dispatch 2024-04-30T09:28:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:38 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c4ea38ab-de25-4ee8-a55c-e75a516ff355"}]': finished 2024-04-30T09:28:38.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:38 smithi046 ceph-mon[23181]: osdmap e27: 5 total, 4 up, 5 in 2024-04-30T09:28:38.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:38 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:39.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:39 smithi026 ceph-mon[27123]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:39.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:39 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/403920402' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:28:39.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:39 smithi026 ceph-mon[22362]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:39.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:39 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/403920402' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:28:39.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:39 smithi046 ceph-mon[23181]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:39.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:39 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/403920402' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:28:41.617 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:41 smithi046 ceph-mon[23181]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:41.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:41 smithi026 ceph-mon[22362]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:41.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:41 smithi026 ceph-mon[27123]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:43.831 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:43 smithi046 ceph-mon[23181]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:43.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:43 smithi026 ceph-mon[22362]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:43 smithi026 ceph-mon[27123]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:44.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-30T09:28:44.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:44 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:44.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:44 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-30T09:28:44.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:44 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:44.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:44 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-30T09:28:44.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:44 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:45.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:45 smithi026 ceph-mon[22362]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:45.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:45 smithi026 ceph-mon[22362]: Deploying daemon osd.4 on smithi046 2024-04-30T09:28:45.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:45 smithi026 ceph-mon[27123]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:45.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:45 smithi026 ceph-mon[27123]: Deploying daemon osd.4 on smithi046 2024-04-30T09:28:45.869 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:45 smithi046 ceph-mon[23181]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:45.869 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:45 smithi046 ceph-mon[23181]: Deploying daemon osd.4 on smithi046 2024-04-30T09:28:47.642 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:47 smithi046 ceph-mon[23181]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:47.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:47 smithi026 ceph-mon[22362]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:47 smithi026 ceph-mon[27123]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:49.866 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:49 smithi046 ceph-mon[23181]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:49.866 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:49 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:49.866 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:49 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:49.866 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:49 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:49.866 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:49 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:49.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[22362]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:49.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:49.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:49.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:49.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:49.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[27123]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:49.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:49.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:49.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:49.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:49 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:51.945 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:51 smithi046 ceph-mon[23181]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:51.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:51 smithi026 ceph-mon[27123]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:51.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:51 smithi026 ceph-mon[22362]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:52.310 INFO:teuthology.orchestra.run.smithi046.stdout:Created osd(s) 4 on host 'smithi046' 2024-04-30T09:28:52.867 DEBUG:teuthology.orchestra.run.smithi046:osd.4> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.4.service 2024-04-30T09:28:52.869 INFO:tasks.cephadm:Deploying osd.5 on smithi046 with /dev/vg_nvme/lv_3... 2024-04-30T09:28:52.869 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-04-30T09:28:53.185 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:52 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:53.185 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:52 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:53.185 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:52 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:53.185 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:52 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:53.185 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:52 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:53.185 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:52 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:53.185 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:52 smithi046 ceph-mon[23181]: from='osd.4 [v2:172.21.15.46:6800/3142607282,v1:172.21.15.46:6801/3142607282]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T09:28:53.185 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:52 smithi046 ceph-mon[23181]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T09:28:53.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:53.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[22362]: from='osd.4 [v2:172.21.15.46:6800/3142607282,v1:172.21.15.46:6801/3142607282]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[22362]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:28:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[27123]: from='osd.4 [v2:172.21.15.46:6800/3142607282,v1:172.21.15.46:6801/3142607282]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T09:28:53.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:52 smithi026 ceph-mon[27123]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T09:28:53.976 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:53 smithi046 ceph-mon[23181]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:53.977 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:53 smithi046 ceph-mon[23181]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-30T09:28:53.977 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:53 smithi046 ceph-mon[23181]: osdmap e28: 5 total, 4 up, 5 in 2024-04-30T09:28:53.977 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:53 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:53.977 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:53 smithi046 ceph-mon[23181]: from='osd.4 [v2:172.21.15.46:6800/3142607282,v1:172.21.15.46:6801/3142607282]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:28:53.977 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:53 smithi046 ceph-mon[23181]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:28:54.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[22362]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[22362]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[22362]: osdmap e28: 5 total, 4 up, 5 in 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[22362]: from='osd.4 [v2:172.21.15.46:6800/3142607282,v1:172.21.15.46:6801/3142607282]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[22362]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[27123]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[27123]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[27123]: osdmap e28: 5 total, 4 up, 5 in 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[27123]: from='osd.4 [v2:172.21.15.46:6800/3142607282,v1:172.21.15.46:6801/3142607282]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:28:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:53 smithi026 ceph-mon[27123]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:28:54.692 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 09:28:54 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[26611]: 2024-04-30T09:28:54.318+0000 7ff8086e6700 -1 osd.4 0 waiting for initial osdmap 2024-04-30T09:28:54.692 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 09:28:54 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[26611]: 2024-04-30T09:28:54.324+0000 7ff80287c700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:28:55.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[27123]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:28:55.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[27123]: osdmap e29: 5 total, 4 up, 5 in 2024-04-30T09:28:55.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:55.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:55.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[27123]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[22362]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:28:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[22362]: osdmap e29: 5 total, 4 up, 5 in 2024-04-30T09:28:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:55 smithi026 ceph-mon[22362]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:55.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:55 smithi046 ceph-mon[23181]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:28:55.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:55 smithi046 ceph-mon[23181]: osdmap e29: 5 total, 4 up, 5 in 2024-04-30T09:28:55.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:55 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:55.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:55 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:55.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:55 smithi046 ceph-mon[23181]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2024-04-30T09:28:56.391 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:56 smithi046 ceph-mon[23181]: purged_snaps scrub starts 2024-04-30T09:28:56.391 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:56 smithi046 ceph-mon[23181]: purged_snaps scrub ok 2024-04-30T09:28:56.391 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:56 smithi046 ceph-mon[23181]: osd.4 [v2:172.21.15.46:6800/3142607282,v1:172.21.15.46:6801/3142607282] boot 2024-04-30T09:28:56.391 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:56 smithi046 ceph-mon[23181]: osdmap e30: 5 total, 5 up, 5 in 2024-04-30T09:28:56.391 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:56.391 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:56 smithi046 ceph-mon[23181]: osdmap e31: 5 total, 5 up, 5 in 2024-04-30T09:28:56.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[27123]: purged_snaps scrub starts 2024-04-30T09:28:56.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[27123]: purged_snaps scrub ok 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[27123]: osd.4 [v2:172.21.15.46:6800/3142607282,v1:172.21.15.46:6801/3142607282] boot 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[27123]: osdmap e30: 5 total, 5 up, 5 in 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[27123]: osdmap e31: 5 total, 5 up, 5 in 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[22362]: purged_snaps scrub starts 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[22362]: purged_snaps scrub ok 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[22362]: osd.4 [v2:172.21.15.46:6800/3142607282,v1:172.21.15.46:6801/3142607282] boot 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[22362]: osdmap e30: 5 total, 5 up, 5 in 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:28:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:56 smithi026 ceph-mon[22362]: osdmap e31: 5 total, 5 up, 5 in 2024-04-30T09:28:57.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi046 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi046 to 17093M 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: osdmap e32: 5 total, 5 up, 5 in 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: Detected new or changed devices on smithi046 2024-04-30T09:28:57.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:57.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:57.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi046 to 17093M 2024-04-30T09:28:57.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:57.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:57.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:28:57.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: osdmap e32: 5 total, 5 up, 5 in 2024-04-30T09:28:57.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:28:57.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:28:57.703 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: Detected new or changed devices on smithi046 2024-04-30T09:28:57.704 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:57.704 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:28:57.704 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi046 to 17093M 2024-04-30T09:28:57.704 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:57.704 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:28:57.704 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:28:57.704 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: osdmap e32: 5 total, 5 up, 5 in 2024-04-30T09:28:57.704 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:28:57.704 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:57 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:28:57.709 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:28:57.729 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi046:vg_nvme/lv_3 2024-04-30T09:28:58.733 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:58 smithi046 ceph-mon[23181]: osdmap e33: 5 total, 5 up, 5 in 2024-04-30T09:28:58.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:58 smithi026 ceph-mon[27123]: osdmap e33: 5 total, 5 up, 5 in 2024-04-30T09:28:58.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:58 smithi026 ceph-mon[22362]: osdmap e33: 5 total, 5 up, 5 in 2024-04-30T09:28:59.733 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:59 smithi046 ceph-mon[23181]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:28:59.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:28:59 smithi046 ceph-mon[23181]: osdmap e34: 5 total, 5 up, 5 in 2024-04-30T09:28:59.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:59 smithi026 ceph-mon[22362]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:28:59.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:28:59 smithi026 ceph-mon[22362]: osdmap e34: 5 total, 5 up, 5 in 2024-04-30T09:28:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:59 smithi026 ceph-mon[27123]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:28:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:28:59 smithi026 ceph-mon[27123]: osdmap e34: 5 total, 5 up, 5 in 2024-04-30T09:29:00.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:00 smithi046 ceph-mon[23181]: from='client.24242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:29:00.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:00 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:29:00.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:00 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:29:00.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:00 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:00.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:00 smithi026 ceph-mon[22362]: from='client.24242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:29:00.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:00 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:29:00.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:00 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:29:00.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:00 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:00 smithi026 ceph-mon[27123]: from='client.24242 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:29:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:00 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:29:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:00 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:29:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:00 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:01.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:01 smithi026 ceph-mon[22362]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-30T09:29:01.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:01 smithi026 ceph-mon[27123]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-30T09:29:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:01 smithi046 ceph-mon[23181]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 103 KiB/s, 0 objects/s recovering 2024-04-30T09:29:02.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/3673456186' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bcfd398-fdfe-4be8-b5ed-da02488b4a3c"}]: dispatch 2024-04-30T09:29:02.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bcfd398-fdfe-4be8-b5ed-da02488b4a3c"}]: dispatch 2024-04-30T09:29:02.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bcfd398-fdfe-4be8-b5ed-da02488b4a3c"}]': finished 2024-04-30T09:29:02.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[27123]: osdmap e35: 6 total, 5 up, 6 in 2024-04-30T09:29:02.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:02.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/3673456186' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bcfd398-fdfe-4be8-b5ed-da02488b4a3c"}]: dispatch 2024-04-30T09:29:02.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bcfd398-fdfe-4be8-b5ed-da02488b4a3c"}]: dispatch 2024-04-30T09:29:02.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bcfd398-fdfe-4be8-b5ed-da02488b4a3c"}]': finished 2024-04-30T09:29:02.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[22362]: osdmap e35: 6 total, 5 up, 6 in 2024-04-30T09:29:02.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:02 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:02.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:02 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/3673456186' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bcfd398-fdfe-4be8-b5ed-da02488b4a3c"}]: dispatch 2024-04-30T09:29:02.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:02 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bcfd398-fdfe-4be8-b5ed-da02488b4a3c"}]: dispatch 2024-04-30T09:29:02.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:02 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bcfd398-fdfe-4be8-b5ed-da02488b4a3c"}]': finished 2024-04-30T09:29:02.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:02 smithi046 ceph-mon[23181]: osdmap e35: 6 total, 5 up, 6 in 2024-04-30T09:29:02.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:02 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:03.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:03 smithi026 ceph-mon[22362]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-30T09:29:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:03 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/1928476900' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:29:03.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:03 smithi026 ceph-mon[27123]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-30T09:29:03.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:03 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/1928476900' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:29:03.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:03 smithi046 ceph-mon[23181]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-30T09:29:03.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:03 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/1928476900' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:29:05.890 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:05 smithi046 ceph-mon[23181]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-30T09:29:05.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:05 smithi026 ceph-mon[22362]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-30T09:29:05.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:05 smithi026 ceph-mon[27123]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-30T09:29:07.792 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:07 smithi046 ceph-mon[23181]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:29:07.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:07 smithi026 ceph-mon[22362]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:29:07.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:07 smithi026 ceph-mon[27123]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:29:08.630 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:08 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-30T09:29:08.630 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:08 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:08.630 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:08 smithi046 ceph-mon[23181]: Deploying daemon osd.5 on smithi046 2024-04-30T09:29:08.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:08 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-30T09:29:08.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:08 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:08.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:08 smithi026 ceph-mon[27123]: Deploying daemon osd.5 on smithi046 2024-04-30T09:29:08.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:08 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-30T09:29:08.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:08 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:08.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:08 smithi026 ceph-mon[22362]: Deploying daemon osd.5 on smithi046 2024-04-30T09:29:09.623 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:09 smithi046 ceph-mon[23181]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-30T09:29:09.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:09 smithi026 ceph-mon[22362]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-30T09:29:09.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:09 smithi026 ceph-mon[27123]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-30T09:29:11.625 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:11 smithi046 ceph-mon[23181]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:11.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:11 smithi026 ceph-mon[22362]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:11.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:11 smithi026 ceph-mon[27123]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[27123]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[22362]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:14 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:14.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:14 smithi046 ceph-mon[23181]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:14.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:14 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:14 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:14 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:14 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:16.224 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:16 smithi046 ceph-mon[23181]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:16.266 INFO:teuthology.orchestra.run.smithi046.stdout:Created osd(s) 5 on host 'smithi046' 2024-04-30T09:29:16.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:16 smithi026 ceph-mon[27123]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:16.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:16 smithi026 ceph-mon[22362]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:16.895 DEBUG:teuthology.orchestra.run.smithi046:osd.5> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.5.service 2024-04-30T09:29:16.898 INFO:tasks.cephadm:Deploying osd.6 on smithi046 with /dev/vg_nvme/lv_2... 2024-04-30T09:29:16.899 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-04-30T09:29:17.144 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:17 smithi046 ceph-mon[23181]: from='osd.5 [v2:172.21.15.46:6808/2766677176,v1:172.21.15.46:6809/2766677176]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T09:29:17.144 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:17 smithi046 ceph-mon[23181]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T09:29:17.144 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:17 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:17.144 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:17 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:17.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:17 smithi026 ceph-mon[22362]: from='osd.5 [v2:172.21.15.46:6808/2766677176,v1:172.21.15.46:6809/2766677176]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T09:29:17.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:17 smithi026 ceph-mon[22362]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T09:29:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:17 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:17 smithi026 ceph-mon[27123]: from='osd.5 [v2:172.21.15.46:6808/2766677176,v1:172.21.15.46:6809/2766677176]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T09:29:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:17 smithi026 ceph-mon[27123]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T09:29:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:17 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:17 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:18.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:18 smithi046 ceph-mon[23181]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:18.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:18 smithi046 ceph-mon[23181]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-30T09:29:18.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:18 smithi046 ceph-mon[23181]: osdmap e36: 6 total, 5 up, 6 in 2024-04-30T09:29:18.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:18 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:18.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:18 smithi046 ceph-mon[23181]: from='osd.5 [v2:172.21.15.46:6808/2766677176,v1:172.21.15.46:6809/2766677176]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:18.320 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:18 smithi046 ceph-mon[23181]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:18.321 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 09:29:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[29494]: 2024-04-30T09:29:18.119+0000 7fc2f50fc700 -1 osd.5 0 waiting for initial osdmap 2024-04-30T09:29:18.321 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 09:29:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[29494]: 2024-04-30T09:29:18.127+0000 7fc2ee290700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:29:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[27123]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[27123]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[27123]: osdmap e36: 6 total, 5 up, 6 in 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[27123]: from='osd.5 [v2:172.21.15.46:6808/2766677176,v1:172.21.15.46:6809/2766677176]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[27123]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[22362]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[22362]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[22362]: osdmap e36: 6 total, 5 up, 6 in 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[22362]: from='osd.5 [v2:172.21.15.46:6808/2766677176,v1:172.21.15.46:6809/2766677176]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:18 smithi026 ceph-mon[22362]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:19 smithi046 ceph-mon[23181]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:29:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:19 smithi046 ceph-mon[23181]: osdmap e37: 6 total, 5 up, 6 in 2024-04-30T09:29:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:19 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:19 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:19.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:19 smithi026 ceph-mon[22362]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:29:19.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:19 smithi026 ceph-mon[22362]: osdmap e37: 6 total, 5 up, 6 in 2024-04-30T09:29:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:19 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:19 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:19 smithi026 ceph-mon[27123]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:29:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:19 smithi026 ceph-mon[27123]: osdmap e37: 6 total, 5 up, 6 in 2024-04-30T09:29:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:19 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:19 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:20.434 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:20 smithi046 ceph-mon[23181]: purged_snaps scrub starts 2024-04-30T09:29:20.434 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:20 smithi046 ceph-mon[23181]: purged_snaps scrub ok 2024-04-30T09:29:20.434 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:20 smithi046 ceph-mon[23181]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:20.434 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:20 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:20.435 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:20 smithi046 ceph-mon[23181]: osd.5 [v2:172.21.15.46:6808/2766677176,v1:172.21.15.46:6809/2766677176] boot 2024-04-30T09:29:20.435 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:20 smithi046 ceph-mon[23181]: osdmap e38: 6 total, 6 up, 6 in 2024-04-30T09:29:20.435 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:20 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:20.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[22362]: purged_snaps scrub starts 2024-04-30T09:29:20.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[22362]: purged_snaps scrub ok 2024-04-30T09:29:20.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[22362]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:20.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[22362]: osd.5 [v2:172.21.15.46:6808/2766677176,v1:172.21.15.46:6809/2766677176] boot 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[22362]: osdmap e38: 6 total, 6 up, 6 in 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[27123]: purged_snaps scrub starts 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[27123]: purged_snaps scrub ok 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[27123]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[27123]: osd.5 [v2:172.21.15.46:6808/2766677176,v1:172.21.15.46:6809/2766677176] boot 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[27123]: osdmap e38: 6 total, 6 up, 6 in 2024-04-30T09:29:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:20 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:29:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: osdmap e39: 6 total, 6 up, 6 in 2024-04-30T09:29:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: Detected new or changed devices on smithi046 2024-04-30T09:29:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi046 to 8546M 2024-04-30T09:29:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: osdmap e39: 6 total, 6 up, 6 in 2024-04-30T09:29:21.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi046 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi046 to 8546M 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: osdmap e39: 6 total, 6 up, 6 in 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: Detected new or changed devices on smithi046 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi046 to 8546M 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:21.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:21.485 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:29:21.505 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi046:vg_nvme/lv_2 2024-04-30T09:29:22.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:22 smithi026 ceph-mon[27123]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:22.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:22 smithi026 ceph-mon[27123]: osdmap e40: 6 total, 6 up, 6 in 2024-04-30T09:29:22.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:22 smithi026 ceph-mon[27123]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-30T09:29:22.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:22 smithi026 ceph-mon[22362]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:22.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:22 smithi026 ceph-mon[22362]: osdmap e40: 6 total, 6 up, 6 in 2024-04-30T09:29:22.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:22 smithi026 ceph-mon[22362]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-30T09:29:22.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:22 smithi046 ceph-mon[23181]: pgmap v131: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:22.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:22 smithi046 ceph-mon[23181]: osdmap e40: 6 total, 6 up, 6 in 2024-04-30T09:29:22.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:22 smithi046 ceph-mon[23181]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-30T09:29:23.395 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:23 smithi046 ceph-mon[23181]: osdmap e41: 6 total, 6 up, 6 in 2024-04-30T09:29:23.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:23 smithi026 ceph-mon[22362]: osdmap e41: 6 total, 6 up, 6 in 2024-04-30T09:29:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:23 smithi026 ceph-mon[27123]: osdmap e41: 6 total, 6 up, 6 in 2024-04-30T09:29:24.404 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:24 smithi046 ceph-mon[23181]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:24.404 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:24 smithi046 ceph-mon[23181]: osdmap e42: 6 total, 6 up, 6 in 2024-04-30T09:29:24.404 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:24 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:29:24.404 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:24 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:29:24.404 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:24 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:24.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[27123]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:24.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[27123]: osdmap e42: 6 total, 6 up, 6 in 2024-04-30T09:29:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:29:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:29:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[22362]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[22362]: osdmap e42: 6 total, 6 up, 6 in 2024-04-30T09:29:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:29:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:29:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:24 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:25.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:25 smithi026 ceph-mon[27123]: from='client.24269 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:29:25.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:25 smithi026 ceph-mon[22362]: from='client.24269 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:29:25.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:25 smithi046 ceph-mon[23181]: from='client.24269 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:29:26.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[22362]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:26.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/441701968' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a2773219-a6c2-414e-b2d0-f381f881c4af"}]: dispatch 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a2773219-a6c2-414e-b2d0-f381f881c4af"}]: dispatch 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a2773219-a6c2-414e-b2d0-f381f881c4af"}]': finished 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[22362]: osdmap e43: 7 total, 6 up, 7 in 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[27123]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/441701968' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a2773219-a6c2-414e-b2d0-f381f881c4af"}]: dispatch 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a2773219-a6c2-414e-b2d0-f381f881c4af"}]: dispatch 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a2773219-a6c2-414e-b2d0-f381f881c4af"}]': finished 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[27123]: osdmap e43: 7 total, 6 up, 7 in 2024-04-30T09:29:26.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:26 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:26 smithi046 ceph-mon[23181]: pgmap v136: 1 pgs: 1 remapped+peering; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:26 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/441701968' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a2773219-a6c2-414e-b2d0-f381f881c4af"}]: dispatch 2024-04-30T09:29:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:26 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a2773219-a6c2-414e-b2d0-f381f881c4af"}]: dispatch 2024-04-30T09:29:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:26 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a2773219-a6c2-414e-b2d0-f381f881c4af"}]': finished 2024-04-30T09:29:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:26 smithi046 ceph-mon[23181]: osdmap e43: 7 total, 6 up, 7 in 2024-04-30T09:29:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:26 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:27.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:27 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/514802669' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:29:27.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:27 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/514802669' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:29:27.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:27 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/514802669' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:29:28.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:28 smithi026 ceph-mon[22362]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-30T09:29:28.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:28 smithi026 ceph-mon[22362]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-30T09:29:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:28 smithi026 ceph-mon[22362]: Cluster is now healthy 2024-04-30T09:29:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:28 smithi026 ceph-mon[27123]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-30T09:29:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:28 smithi026 ceph-mon[27123]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-30T09:29:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:28 smithi026 ceph-mon[27123]: Cluster is now healthy 2024-04-30T09:29:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:28 smithi046 ceph-mon[23181]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-30T09:29:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:28 smithi046 ceph-mon[23181]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-30T09:29:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:28 smithi046 ceph-mon[23181]: Cluster is now healthy 2024-04-30T09:29:30.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:30 smithi026 ceph-mon[22362]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-30T09:29:30.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:30 smithi026 ceph-mon[27123]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-30T09:29:30.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:30 smithi046 ceph-mon[23181]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-30T09:29:32.394 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:32 smithi046 ceph-mon[23181]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:29:32.394 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:32 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-30T09:29:32.394 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:32 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:32.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:32 smithi026 ceph-mon[22362]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:29:32.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-30T09:29:32.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:32 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:32.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:32 smithi026 ceph-mon[27123]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:29:32.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:32 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-30T09:29:32.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:32 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:33.273 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:33 smithi046 ceph-mon[23181]: Deploying daemon osd.6 on smithi046 2024-04-30T09:29:33.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:33 smithi026 ceph-mon[22362]: Deploying daemon osd.6 on smithi046 2024-04-30T09:29:33.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:33 smithi026 ceph-mon[27123]: Deploying daemon osd.6 on smithi046 2024-04-30T09:29:34.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:34 smithi046 ceph-mon[23181]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-30T09:29:34.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:34 smithi026 ceph-mon[22362]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-30T09:29:34.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:34 smithi026 ceph-mon[27123]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 55 KiB/s, 0 objects/s recovering 2024-04-30T09:29:36.371 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:36 smithi046 ceph-mon[23181]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-30T09:29:36.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:36 smithi026 ceph-mon[22362]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-30T09:29:36.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:36 smithi026 ceph-mon[27123]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-30T09:29:38.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[22362]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[27123]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:37 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:38.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:37 smithi046 ceph-mon[23181]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:37 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:40.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:39 smithi026 ceph-mon[22362]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:39 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:39 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:40.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:39 smithi026 ceph-mon[27123]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:40.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:39 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:40.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:39 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:39 smithi046 ceph-mon[23181]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:39 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:39 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:40.301 INFO:teuthology.orchestra.run.smithi046.stdout:Created osd(s) 6 on host 'smithi046' 2024-04-30T09:29:40.936 DEBUG:teuthology.orchestra.run.smithi046:osd.6> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.6.service 2024-04-30T09:29:40.938 INFO:tasks.cephadm:Deploying osd.7 on smithi046 with /dev/vg_nvme/lv_1... 2024-04-30T09:29:40.938 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-04-30T09:29:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[27123]: from='osd.6 [v2:172.21.15.46:6816/703785050,v1:172.21.15.46:6817/703785050]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[27123]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[22362]: from='osd.6 [v2:172.21.15.46:6816/703785050,v1:172.21.15.46:6817/703785050]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[22362]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:40 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:40 smithi046 ceph-mon[23181]: from='osd.6 [v2:172.21.15.46:6816/703785050,v1:172.21.15.46:6817/703785050]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T09:29:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:40 smithi046 ceph-mon[23181]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T09:29:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:40 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:40 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:29:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:40 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:40 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:29:42.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:41 smithi046 ceph-mon[23181]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:42.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:41 smithi046 ceph-mon[23181]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-30T09:29:42.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:41 smithi046 ceph-mon[23181]: osdmap e44: 7 total, 6 up, 7 in 2024-04-30T09:29:42.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:41 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:42.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:41 smithi046 ceph-mon[23181]: from='osd.6 [v2:172.21.15.46:6816/703785050,v1:172.21.15.46:6817/703785050]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:42.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:41 smithi046 ceph-mon[23181]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:42.062 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 09:29:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[32379]: 2024-04-30T09:29:41.953+0000 7f6abfe4c700 -1 osd.6 0 waiting for initial osdmap 2024-04-30T09:29:42.062 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 09:29:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[32379]: 2024-04-30T09:29:41.961+0000 7f6aba7e3700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:29:42.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[22362]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:42.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[22362]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[22362]: osdmap e44: 7 total, 6 up, 7 in 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[22362]: from='osd.6 [v2:172.21.15.46:6816/703785050,v1:172.21.15.46:6817/703785050]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[22362]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[27123]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[27123]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[27123]: osdmap e44: 7 total, 6 up, 7 in 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[27123]: from='osd.6 [v2:172.21.15.46:6816/703785050,v1:172.21.15.46:6817/703785050]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:41 smithi026 ceph-mon[27123]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:29:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:42 smithi046 ceph-mon[23181]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:29:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:42 smithi046 ceph-mon[23181]: osdmap e45: 7 total, 6 up, 7 in 2024-04-30T09:29:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:42 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:42 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:43.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:42 smithi026 ceph-mon[22362]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:29:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:42 smithi026 ceph-mon[22362]: osdmap e45: 7 total, 6 up, 7 in 2024-04-30T09:29:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:42 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:43.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:42 smithi026 ceph-mon[27123]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:29:43.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:42 smithi026 ceph-mon[27123]: osdmap e45: 7 total, 6 up, 7 in 2024-04-30T09:29:43.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:43.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:42 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:44.212 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:43 smithi046 ceph-mon[23181]: purged_snaps scrub starts 2024-04-30T09:29:44.212 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:43 smithi046 ceph-mon[23181]: purged_snaps scrub ok 2024-04-30T09:29:44.212 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:43 smithi046 ceph-mon[23181]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:44.212 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:43 smithi046 ceph-mon[23181]: osd.6 [v2:172.21.15.46:6816/703785050,v1:172.21.15.46:6817/703785050] boot 2024-04-30T09:29:44.212 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:43 smithi046 ceph-mon[23181]: osdmap e46: 7 total, 7 up, 7 in 2024-04-30T09:29:44.212 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:43 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:44.212 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:43 smithi046 ceph-mon[23181]: osdmap e47: 7 total, 7 up, 7 in 2024-04-30T09:29:44.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[22362]: purged_snaps scrub starts 2024-04-30T09:29:44.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[22362]: purged_snaps scrub ok 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[22362]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[22362]: osd.6 [v2:172.21.15.46:6816/703785050,v1:172.21.15.46:6817/703785050] boot 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[22362]: osdmap e46: 7 total, 7 up, 7 in 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[22362]: osdmap e47: 7 total, 7 up, 7 in 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[27123]: purged_snaps scrub starts 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[27123]: purged_snaps scrub ok 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[27123]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[27123]: osd.6 [v2:172.21.15.46:6816/703785050,v1:172.21.15.46:6817/703785050] boot 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[27123]: osdmap e46: 7 total, 7 up, 7 in 2024-04-30T09:29:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:29:44.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:43 smithi026 ceph-mon[27123]: osdmap e47: 7 total, 7 up, 7 in 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: Detected new or changed devices on smithi046 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi046 to 5697M 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: osdmap e48: 7 total, 7 up, 7 in 2024-04-30T09:29:45.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:45 smithi046 ceph-mon[23181]: osdmap e49: 7 total, 7 up, 7 in 2024-04-30T09:29:46.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:29:46.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi046 2024-04-30T09:29:46.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:46.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi046 to 5697M 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: osdmap e48: 7 total, 7 up, 7 in 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[27123]: osdmap e49: 7 total, 7 up, 7 in 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: Detected new or changed devices on smithi046 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:29:46.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi046 to 5697M 2024-04-30T09:29:46.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:46.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:29:46.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: osdmap e48: 7 total, 7 up, 7 in 2024-04-30T09:29:46.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:45 smithi026 ceph-mon[22362]: osdmap e49: 7 total, 7 up, 7 in 2024-04-30T09:29:46.348 INFO:teuthology.orchestra.run.smithi046.stdout: 2024-04-30T09:29:46.371 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch daemon add osd smithi046:vg_nvme/lv_1 2024-04-30T09:29:47.622 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:47 smithi046 ceph-mon[23181]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:29:47.623 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:47 smithi046 ceph-mon[23181]: osdmap e50: 7 total, 7 up, 7 in 2024-04-30T09:29:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:47 smithi026 ceph-mon[27123]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:29:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:47 smithi026 ceph-mon[27123]: osdmap e50: 7 total, 7 up, 7 in 2024-04-30T09:29:47.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:47 smithi026 ceph-mon[22362]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:29:47.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:47 smithi026 ceph-mon[22362]: osdmap e50: 7 total, 7 up, 7 in 2024-04-30T09:29:48.676 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:29:48.676 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:29:48.676 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:48 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:48.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:29:48.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:29:48.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:48 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:48.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-30T09:29:48.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-30T09:29:48.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:48 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:49.699 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:49 smithi046 ceph-mon[23181]: from='client.24296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:29:49.699 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:49 smithi046 ceph-mon[23181]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-04-30T09:29:49.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:49 smithi026 ceph-mon[22362]: from='client.24296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:29:49.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:49 smithi026 ceph-mon[22362]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-04-30T09:29:49.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:49 smithi026 ceph-mon[27123]: from='client.24296 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi046:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:29:49.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:49 smithi026 ceph-mon[27123]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-04-30T09:29:51.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[22362]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/3824214528' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04ed7db9-9f70-4d69-8e14-2f738d07bfdd"}]: dispatch 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04ed7db9-9f70-4d69-8e14-2f738d07bfdd"}]: dispatch 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[22362]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04ed7db9-9f70-4d69-8e14-2f738d07bfdd"}]': finished 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[22362]: osdmap e51: 8 total, 7 up, 8 in 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/3375056937' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[27123]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/3824214528' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04ed7db9-9f70-4d69-8e14-2f738d07bfdd"}]: dispatch 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04ed7db9-9f70-4d69-8e14-2f738d07bfdd"}]: dispatch 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[27123]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04ed7db9-9f70-4d69-8e14-2f738d07bfdd"}]': finished 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[27123]: osdmap e51: 8 total, 7 up, 8 in 2024-04-30T09:29:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:29:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:51 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/3375056937' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:29:51.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:51 smithi046 ceph-mon[23181]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2024-04-30T09:29:51.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:51 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/3824214528' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04ed7db9-9f70-4d69-8e14-2f738d07bfdd"}]: dispatch 2024-04-30T09:29:51.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:51 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04ed7db9-9f70-4d69-8e14-2f738d07bfdd"}]: dispatch 2024-04-30T09:29:51.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:51 smithi046 ceph-mon[23181]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04ed7db9-9f70-4d69-8e14-2f738d07bfdd"}]': finished 2024-04-30T09:29:51.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:51 smithi046 ceph-mon[23181]: osdmap e51: 8 total, 7 up, 8 in 2024-04-30T09:29:51.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:51 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:29:51.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:51 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/3375056937' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-30T09:29:53.923 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:53 smithi046 ceph-mon[23181]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-30T09:29:53.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:53 smithi026 ceph-mon[22362]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-30T09:29:53.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:53 smithi026 ceph-mon[27123]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2024-04-30T09:29:55.656 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:55 smithi046 ceph-mon[23181]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:29:55.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:55 smithi026 ceph-mon[22362]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:29:55.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:55 smithi026 ceph-mon[27123]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:29:56.879 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:29:56.880 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:56 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:29:56.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:29:56.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:56 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:29:56.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:29:56.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:56 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:29:57.633 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:57 smithi046 ceph-mon[23181]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-30T09:29:57.633 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:57 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-30T09:29:57.633 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:57 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:57.633 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:57 smithi046 ceph-mon[23181]: Deploying daemon osd.7 on smithi046 2024-04-30T09:29:57.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:57 smithi026 ceph-mon[27123]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-30T09:29:57.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:57 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-30T09:29:57.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:57 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:57.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:57 smithi026 ceph-mon[27123]: Deploying daemon osd.7 on smithi046 2024-04-30T09:29:57.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:57 smithi026 ceph-mon[22362]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-30T09:29:57.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-30T09:29:57.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:57 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:29:57.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:57 smithi026 ceph-mon[22362]: Deploying daemon osd.7 on smithi046 2024-04-30T09:29:59.717 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:29:59 smithi046 ceph-mon[23181]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:29:59.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:29:59 smithi026 ceph-mon[22362]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:29:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:29:59 smithi026 ceph-mon[27123]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:00 smithi046 ceph-mon[23181]: overall HEALTH_OK 2024-04-30T09:30:00.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:00 smithi026 ceph-mon[22362]: overall HEALTH_OK 2024-04-30T09:30:00.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:00 smithi026 ceph-mon[27123]: overall HEALTH_OK 2024-04-30T09:30:01.833 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:01 smithi046 ceph-mon[23181]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:01.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:01 smithi026 ceph-mon[22362]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:01.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:01 smithi026 ceph-mon[27123]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:03.378 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:03 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:03.378 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:03 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:03.378 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:03 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:03.378 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:03 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:03.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:03 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:03.472 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:03 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:03.472 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:03 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:03.472 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:03 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:03.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:03 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:03.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:03 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:03.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:03 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:03.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:03 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:04.398 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:04 smithi046 ceph-mon[23181]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:04 smithi026 ceph-mon[27123]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:04 smithi026 ceph-mon[22362]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:05.162 INFO:teuthology.orchestra.run.smithi046.stdout:Created osd(s) 7 on host 'smithi046' 2024-04-30T09:30:05.878 DEBUG:teuthology.orchestra.run.smithi046:osd.7> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.7.service 2024-04-30T09:30:05.881 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-30T09:30:05.881 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-30T09:30:06.434 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:06 smithi046 ceph-mon[23181]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:06.434 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:06 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:06.435 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:06 smithi046 ceph-mon[23181]: from='osd.7 [v2:172.21.15.46:6824/4197915406,v1:172.21.15.46:6825/4197915406]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T09:30:06.435 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:06 smithi046 ceph-mon[23181]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T09:30:06.435 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:06 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:06.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[22362]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:06.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:06.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[22362]: from='osd.7 [v2:172.21.15.46:6824/4197915406,v1:172.21.15.46:6825/4197915406]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T09:30:06.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[22362]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T09:30:06.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[27123]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[27123]: from='osd.7 [v2:172.21.15.46:6824/4197915406,v1:172.21.15.46:6825/4197915406]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T09:30:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[27123]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T09:30:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:06 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:07.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[27123]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-30T09:30:07.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[27123]: osdmap e52: 8 total, 7 up, 8 in 2024-04-30T09:30:07.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:07.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[27123]: from='osd.7 [v2:172.21.15.46:6824/4197915406,v1:172.21.15.46:6825/4197915406]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:30:07.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[27123]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:30:07.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[22362]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-30T09:30:07.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[22362]: osdmap e52: 8 total, 7 up, 8 in 2024-04-30T09:30:07.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:07.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[22362]: from='osd.7 [v2:172.21.15.46:6824/4197915406,v1:172.21.15.46:6825/4197915406]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:30:07.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:07 smithi026 ceph-mon[22362]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:30:07.438 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:07 smithi046 ceph-mon[23181]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-30T09:30:07.438 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:07 smithi046 ceph-mon[23181]: osdmap e52: 8 total, 7 up, 8 in 2024-04-30T09:30:07.438 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:07 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:07.438 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:07 smithi046 ceph-mon[23181]: from='osd.7 [v2:172.21.15.46:6824/4197915406,v1:172.21.15.46:6825/4197915406]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:30:07.439 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:07 smithi046 ceph-mon[23181]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T09:30:07.439 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 09:30:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[35380]: 2024-04-30T09:30:07.173+0000 7fb6ab05f700 -1 osd.7 0 waiting for initial osdmap 2024-04-30T09:30:07.439 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 09:30:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[35380]: 2024-04-30T09:30:07.183+0000 7fb6a39f2700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:30:08.082 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:30:08.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[27123]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:08.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[27123]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:30:08.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[27123]: osdmap e53: 8 total, 7 up, 8 in 2024-04-30T09:30:08.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:08.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:08.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3399118688' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-30T09:30:08.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[22362]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:08.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[22362]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:30:08.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[22362]: osdmap e53: 8 total, 7 up, 8 in 2024-04-30T09:30:08.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:08.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:08.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:08 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3399118688' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-30T09:30:08.424 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:08 smithi046 ceph-mon[23181]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2024-04-30T09:30:08.424 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:08 smithi046 ceph-mon[23181]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]': finished 2024-04-30T09:30:08.424 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:08 smithi046 ceph-mon[23181]: osdmap e53: 8 total, 7 up, 8 in 2024-04-30T09:30:08.424 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:08 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:08.424 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:08 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:08.424 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:08 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3399118688' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-30T09:30:08.759 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1714469382,"num_in_osds":8,"osd_in_since":1714469390,"num_remapped_pgs":0} 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: purged_snaps scrub starts 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: purged_snaps scrub ok 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: osd.7 [v2:172.21.15.46:6824/4197915406,v1:172.21.15.46:6825/4197915406] boot 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: osdmap e54: 8 total, 8 up, 8 in 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:09.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:09.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:09.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: purged_snaps scrub starts 2024-04-30T09:30:09.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: purged_snaps scrub ok 2024-04-30T09:30:09.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: osd.7 [v2:172.21.15.46:6824/4197915406,v1:172.21.15.46:6825/4197915406] boot 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: osdmap e54: 8 total, 8 up, 8 in 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:09.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:09 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: purged_snaps scrub starts 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: purged_snaps scrub ok 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: osd.7 [v2:172.21.15.46:6824/4197915406,v1:172.21.15.46:6825/4197915406] boot 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: osdmap e54: 8 total, 8 up, 8 in 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:09 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:09.760 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd stat -f json 2024-04-30T09:30:10.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:10 smithi026 ceph-mon[27123]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:10.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:10 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi046 2024-04-30T09:30:10.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:10 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi046 to 4273M 2024-04-30T09:30:10.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:10 smithi026 ceph-mon[27123]: osdmap e55: 8 total, 8 up, 8 in 2024-04-30T09:30:10.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:10 smithi026 ceph-mon[22362]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:10.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:10 smithi026 ceph-mon[22362]: Detected new or changed devices on smithi046 2024-04-30T09:30:10.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:10 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi046 to 4273M 2024-04-30T09:30:10.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:10 smithi026 ceph-mon[22362]: osdmap e55: 8 total, 8 up, 8 in 2024-04-30T09:30:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:10 smithi046 ceph-mon[23181]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 46 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:10 smithi046 ceph-mon[23181]: Detected new or changed devices on smithi046 2024-04-30T09:30:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:10 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi046 to 4273M 2024-04-30T09:30:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:10 smithi046 ceph-mon[23181]: osdmap e55: 8 total, 8 up, 8 in 2024-04-30T09:30:11.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:11 smithi026 ceph-mon[27123]: osdmap e56: 8 total, 8 up, 8 in 2024-04-30T09:30:11.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:11 smithi026 ceph-mon[27123]: osdmap e57: 8 total, 8 up, 8 in 2024-04-30T09:30:11.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:11 smithi026 ceph-mon[22362]: osdmap e56: 8 total, 8 up, 8 in 2024-04-30T09:30:11.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:11 smithi026 ceph-mon[22362]: osdmap e57: 8 total, 8 up, 8 in 2024-04-30T09:30:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:11 smithi046 ceph-mon[23181]: osdmap e56: 8 total, 8 up, 8 in 2024-04-30T09:30:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:11 smithi046 ceph-mon[23181]: osdmap e57: 8 total, 8 up, 8 in 2024-04-30T09:30:12.038 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:30:12.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:12 smithi026 ceph-mon[22362]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:12.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:12 smithi026 ceph-mon[22362]: osdmap e58: 8 total, 8 up, 8 in 2024-04-30T09:30:12.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:12 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3436132087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-30T09:30:12.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:12 smithi026 ceph-mon[27123]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:12.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:12 smithi026 ceph-mon[27123]: osdmap e58: 8 total, 8 up, 8 in 2024-04-30T09:30:12.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:12 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3436132087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-30T09:30:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:12 smithi046 ceph-mon[23181]: pgmap v173: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:12 smithi046 ceph-mon[23181]: osdmap e58: 8 total, 8 up, 8 in 2024-04-30T09:30:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:12 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3436132087' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-30T09:30:12.719 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1714469408,"num_in_osds":8,"osd_in_since":1714469390,"num_remapped_pgs":0} 2024-04-30T09:30:12.719 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-30T09:30:12.945 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:14.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:14 smithi026 ceph-mon[27123]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:14.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:14 smithi026 ceph-mon[22362]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:14 smithi046 ceph-mon[23181]: pgmap v176: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:14.782 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:30:14.782 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":58,"fsid":"703f4aea-06d3-11ef-bc93-c7b262605968","created":"2024-04-30T09:25:00.900233+0000","modified":"2024-04-30T09:30:11.519033+0000","last_up_change":"2024-04-30T09:30:08.170335+0000","last_in_change":"2024-04-30T09:29:50.713788+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-30T09:28:08.974604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"031c2da1-c3a9-4d2b-83fe-93835cc5295a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6803","nonce":2102761558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6805","nonce":2102761558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6809","nonce":2102761558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6807","nonce":2102761558}]},"public_addr":"172.21.15.26:6803/2102761558","cluster_addr":"172.21.15.26:6805/2102761558","heartbeat_back_addr":"172.21.15.26:6809/2102761558","heartbeat_front_addr":"172.21.15.26:6807/2102761558","state":["exists","up"]},{"osd":1,"uuid":"59928493-d00b-4259-b638-8b4743f1434e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6811","nonce":1899832674}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6813","nonce":1899832674}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6817","nonce":1899832674}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6815","nonce":1899832674}]},"public_addr":"172.21.15.26:6811/1899832674","cluster_addr":"172.21.15.26:6813/1899832674","heartbeat_back_addr":"172.21.15.26:6817/1899832674","heartbeat_front_addr":"172.21.15.26:6815/1899832674","state":["exists","up"]},{"osd":2,"uuid":"07cad5bf-069c-4688-8de5-abcf2b56f970","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6819","nonce":3754664132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6821","nonce":3754664132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6825","nonce":3754664132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6823","nonce":3754664132}]},"public_addr":"172.21.15.26:6819/3754664132","cluster_addr":"172.21.15.26:6821/3754664132","heartbeat_back_addr":"172.21.15.26:6825/3754664132","heartbeat_front_addr":"172.21.15.26:6823/3754664132","state":["exists","up"]},{"osd":3,"uuid":"5e016838-5979-4375-aae0-8a42a81a5290","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6827","nonce":1106154924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6829","nonce":1106154924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6833","nonce":1106154924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6831","nonce":1106154924}]},"public_addr":"172.21.15.26:6827/1106154924","cluster_addr":"172.21.15.26:6829/1106154924","heartbeat_back_addr":"172.21.15.26:6833/1106154924","heartbeat_front_addr":"172.21.15.26:6831/1106154924","state":["exists","up"]},{"osd":4,"uuid":"c4ea38ab-de25-4ee8-a55c-e75a516ff355","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6800","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6801","nonce":3142607282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6802","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6803","nonce":3142607282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6806","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6807","nonce":3142607282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6804","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6805","nonce":3142607282}]},"public_addr":"172.21.15.46:6801/3142607282","cluster_addr":"172.21.15.46:6803/3142607282","heartbeat_back_addr":"172.21.15.46:6807/3142607282","heartbeat_front_addr":"172.21.15.46:6805/3142607282","state":["exists","up"]},{"osd":5,"uuid":"6bcfd398-fdfe-4be8-b5ed-da02488b4a3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6808","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6809","nonce":2766677176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6810","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6811","nonce":2766677176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6814","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6815","nonce":2766677176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6812","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6813","nonce":2766677176}]},"public_addr":"172.21.15.46:6809/2766677176","cluster_addr":"172.21.15.46:6811/2766677176","heartbeat_back_addr":"172.21.15.46:6815/2766677176","heartbeat_front_addr":"172.21.15.46:6813/2766677176","state":["exists","up"]},{"osd":6,"uuid":"a2773219-a6c2-414e-b2d0-f381f881c4af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6816","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6817","nonce":703785050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6818","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6819","nonce":703785050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6822","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6823","nonce":703785050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6820","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6821","nonce":703785050}]},"public_addr":"172.21.15.46:6817/703785050","cluster_addr":"172.21.15.46:6819/703785050","heartbeat_back_addr":"172.21.15.46:6823/703785050","heartbeat_front_addr":"172.21.15.46:6821/703785050","state":["exists","up"]},{"osd":7,"uuid":"04ed7db9-9f70-4d69-8e14-2f738d07bfdd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6824","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6825","nonce":4197915406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6826","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6827","nonce":4197915406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6830","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6831","nonce":4197915406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6828","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6829","nonce":4197915406}]},"public_addr":"172.21.15.46:6825/4197915406","cluster_addr":"172.21.15.46:6827/4197915406","heartbeat_back_addr":"172.21.15.46:6831/4197915406","heartbeat_front_addr":"172.21.15.46:6829/4197915406","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:27:16.963970+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:27:40.868367+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:28:05.980512+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:28:31.450304+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:28:53.337778+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:29:17.234061+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:29:41.061817+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:30:06.161935+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/3683337897":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/239085219":"2024-05-01T09:25:55.470596+0000","172.21.15.26:6801/1844994599":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/3072786577":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/514779481":"2024-05-01T09:25:32.971742+0000","172.21.15.26:0/2113986196":"2024-05-01T09:25:32.971742+0000","172.21.15.26:6801/733429943":"2024-05-01T09:25:32.971742+0000","172.21.15.26:6800/733429943":"2024-05-01T09:25:32.971742+0000","172.21.15.26:6800/1844994599":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/4095183837":"2024-05-01T09:25:32.971742+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-30T09:30:15.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:15 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3303741003' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-30T09:30:15.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:15 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3303741003' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-30T09:30:15.459 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-30T09:28:08.974604+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-04-30T09:30:15.459 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-30T09:30:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:15 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3303741003' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-30T09:30:15.686 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:16.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:16 smithi026 ceph-mon[22362]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-04-30T09:30:16.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:16 smithi026 ceph-mon[27123]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-04-30T09:30:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:16 smithi046 ceph-mon[23181]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-04-30T09:30:17.538 INFO:teuthology.orchestra.run.smithi026.stdout:pg_num: 1 2024-04-30T09:30:18.191 INFO:tasks.cephadm:Adding prometheus.a on smithi046 2024-04-30T09:30:18.191 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch apply prometheus '1;smithi046=a' 2024-04-30T09:30:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:18 smithi026 ceph-mon[27123]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-30T09:30:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:18 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3762944138' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-30T09:30:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:18 smithi026 ceph-mon[22362]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-30T09:30:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:18 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3762944138' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-30T09:30:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:18 smithi046 ceph-mon[23181]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-04-30T09:30:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:18 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3762944138' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-30T09:30:20.112 INFO:teuthology.orchestra.run.smithi046.stdout:Scheduled prometheus update... 2024-04-30T09:30:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:20 smithi046 ceph-mon[23181]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:30:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:20 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:20 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:20 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:20 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:20 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:20 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-30T09:30:20.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[22362]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:30:20.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:20.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:20.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[27123]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' 2024-04-30T09:30:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:20 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-30T09:30:20.727 DEBUG:teuthology.orchestra.run.smithi046:prometheus.a> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service 2024-04-30T09:30:20.730 INFO:tasks.cephadm:Adding node-exporter.a on smithi026 2024-04-30T09:30:20.730 INFO:tasks.cephadm:Adding node-exporter.b on smithi046 2024-04-30T09:30:20.730 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch apply node-exporter '2;smithi026=a;smithi046=b' 2024-04-30T09:30:21.420 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-mon[23181]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi046=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:21.420 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-mon[23181]: Saving service prometheus spec with placement smithi046=a;count:1 2024-04-30T09:30:21.420 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-mon[23181]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-30T09:30:21.421 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-mon[23181]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-30T09:30:21.421 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ignoring --setuser ceph since I am not root 2024-04-30T09:30:21.421 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ignoring --setgroup ceph since I am not root 2024-04-30T09:30:21.421 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-mgr[24375]: -- 172.21.15.46:0/1348941433 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f4e0fa2340 con 0x55f4e0f7a800 2024-04-30T09:30:21.421 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:21.290+0000 7f8903b76000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-30T09:30:21.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-mon[22362]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi046=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:21.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-mon[22362]: Saving service prometheus spec with placement smithi046=a;count:1 2024-04-30T09:30:21.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-mon[22362]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-30T09:30:21.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-mon[22362]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-30T09:30:21.438 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-mon[27123]: from='client.24338 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi046=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:21.438 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-mon[27123]: Saving service prometheus spec with placement smithi046=a;count:1 2024-04-30T09:30:21.438 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-mon[27123]: from='mgr.14152 172.21.15.26:0/2294872313' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-30T09:30:21.438 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-mon[27123]: mgrmap e16: y(active, since 4m), standbys: x 2024-04-30T09:30:21.438 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ignoring --setuser ceph since I am not root 2024-04-30T09:30:21.438 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ignoring --setgroup ceph since I am not root 2024-04-30T09:30:21.438 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-mgr[22595]: -- 172.21.15.26:0/3110243804 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55922ecf4340 con 0x55922eccc800 2024-04-30T09:30:21.439 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:21.302+0000 7f7cd2d3c000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-30T09:30:21.439 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:21.436+0000 7f7cd2d3c000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:30:21.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:21.538+0000 7f7cd2d3c000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-30T09:30:21.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:21.427+0000 7f8903b76000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:30:21.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:21.529+0000 7f8903b76000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-30T09:30:22.040 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:21.851+0000 7f8903b76000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-30T09:30:22.040 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:21.936+0000 7f8903b76000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-30T09:30:22.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:21.857+0000 7f7cd2d3c000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-30T09:30:22.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:21.943+0000 7f7cd2d3c000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-30T09:30:23.389 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:23.131+0000 7f7cd2d3c000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-30T09:30:23.389 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:23.302+0000 7f7cd2d3c000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-30T09:30:23.411 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:23.154+0000 7f8903b76000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-30T09:30:23.411 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:23.325+0000 7f8903b76000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-30T09:30:23.411 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:23.410+0000 7f8903b76000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-30T09:30:23.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:23.388+0000 7f7cd2d3c000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-30T09:30:24.163 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:23.715+0000 7f7cd2d3c000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-30T09:30:24.163 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:23.810+0000 7f7cd2d3c000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-30T09:30:24.163 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:23.906+0000 7f7cd2d3c000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-30T09:30:24.178 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:23.736+0000 7f8903b76000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-30T09:30:24.178 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:23.830+0000 7f8903b76000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-30T09:30:24.178 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:23.926+0000 7f8903b76000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-30T09:30:24.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:24 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:24.162+0000 7f7cd2d3c000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:30:24.483 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:24.177+0000 7f8903b76000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:30:24.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:24 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:24.627+0000 7f7cd2d3c000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-30T09:30:24.983 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:24.640+0000 7f8903b76000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-30T09:30:25.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:25 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:25.135+0000 7f7cd2d3c000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-30T09:30:25.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:25 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:25.234+0000 7f7cd2d3c000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-30T09:30:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:25.143+0000 7f8903b76000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-30T09:30:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:25.241+0000 7f8903b76000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-30T09:30:25.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:25 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:25.545+0000 7f7cd2d3c000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-30T09:30:25.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:25 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:25.638+0000 7f7cd2d3c000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-30T09:30:25.983 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:25.549+0000 7f8903b76000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-30T09:30:25.983 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:25.642+0000 7f8903b76000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-30T09:30:26.415 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:26.060+0000 7f8903b76000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-30T09:30:26.415 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:26.199+0000 7f8903b76000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-30T09:30:26.415 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:26.295+0000 7f8903b76000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-30T09:30:26.415 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:26.414+0000 7f8903b76000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-30T09:30:26.417 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:26.060+0000 7f7cd2d3c000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-30T09:30:26.418 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:26.200+0000 7f7cd2d3c000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-30T09:30:26.418 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:26.297+0000 7f7cd2d3c000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-30T09:30:26.418 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:26.416+0000 7f7cd2d3c000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-30T09:30:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:26.511+0000 7f7cd2d3c000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-30T09:30:26.733 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:26.507+0000 7f8903b76000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-30T09:30:27.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:27.089+0000 7f7cd2d3c000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-30T09:30:27.453 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:27.080+0000 7f8903b76000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-30T09:30:27.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-mon[23181]: Standby manager daemon x restarted 2024-04-30T09:30:27.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-mon[23181]: Standby manager daemon x started 2024-04-30T09:30:27.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:30:27.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-mon[23181]: Active manager daemon y restarted 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-mon[23181]: Activating manager daemon y 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-mon[23181]: osdmap e59: 8 total, 8 up, 8 in 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:27.452+0000 7f8903b76000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: [30/Apr/2024:09:30:27] ENGINE Bus STARTING 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: CherryPy Checker: 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: The Application mounted at '' has an empty config. 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: 2024-04-30T09:30:27.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: [30/Apr/2024:09:30:27] ENGINE Serving on http://:::9283 2024-04-30T09:30:27.736 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: [30/Apr/2024:09:30:27] ENGINE Bus STARTED 2024-04-30T09:30:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[22362]: Standby manager daemon x restarted 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[22362]: Standby manager daemon x started 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[22362]: Active manager daemon y restarted 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[22362]: Activating manager daemon y 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[22362]: osdmap e59: 8 total, 8 up, 8 in 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[27123]: Standby manager daemon x restarted 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[27123]: Standby manager daemon x started 2024-04-30T09:30:27.755 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/604142210' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[27123]: Active manager daemon y restarted 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[27123]: Activating manager daemon y 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-mon[27123]: osdmap e59: 8 total, 8 up, 8 in 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:27.466+0000 7f7cd2d3c000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:30:27] ENGINE Bus STARTING 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: CherryPy Checker: 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: The Application mounted at '' has an empty config. 2024-04-30T09:30:27.756 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:30:27.757 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:30:27] ENGINE Serving on http://:::9283 2024-04-30T09:30:27.757 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:30:27] ENGINE Bus STARTED 2024-04-30T09:30:28.005 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:30:27] ENGINE Bus STARTING 2024-04-30T09:30:28.006 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:30:28] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:30:28.359 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:30:28] ENGINE Bus STARTED 2024-04-30T09:30:28.534 INFO:teuthology.orchestra.run.smithi046.stdout:Scheduled node-exporter update... 2024-04-30T09:30:28.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: mgrmap e17: y(active, starting, since 0.0408916s), standbys: x 2024-04-30T09:30:28.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:30:28.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:30:28.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:30:28.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:30:28.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:30:28.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: Manager daemon y is now available 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:28.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: [30/Apr/2024:09:30:27] ENGINE Bus STARTING 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: [30/Apr/2024:09:30:28] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: [30/Apr/2024:09:30:28] ENGINE Bus STARTED 2024-04-30T09:30:28.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:28.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: mgrmap e17: y(active, starting, since 0.0408916s), standbys: x 2024-04-30T09:30:28.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:30:28.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:30:28.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:30:28.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:30:28.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:30:28.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:30:28.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:30:28.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: Manager daemon y is now available 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:28.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:28.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:30:28.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:30:28.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:30:28.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:30:28.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: [30/Apr/2024:09:30:27] ENGINE Bus STARTING 2024-04-30T09:30:28.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: [30/Apr/2024:09:30:28] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:30:28.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: [30/Apr/2024:09:30:28] ENGINE Bus STARTED 2024-04-30T09:30:28.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:28.726 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: mgrmap e17: y(active, starting, since 0.0408916s), standbys: x 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:30:28.727 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: Manager daemon y is now available 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:30:28.728 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: [30/Apr/2024:09:30:27] ENGINE Bus STARTING 2024-04-30T09:30:28.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: [30/Apr/2024:09:30:28] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:30:28.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: [30/Apr/2024:09:30:28] ENGINE Bus STARTED 2024-04-30T09:30:28.729 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:29.326 DEBUG:teuthology.orchestra.run.smithi026:node-exporter.a> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service 2024-04-30T09:30:29.329 DEBUG:teuthology.orchestra.run.smithi046:node-exporter.b> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service 2024-04-30T09:30:29.333 INFO:tasks.cephadm:Adding alertmanager.a on smithi026 2024-04-30T09:30:29.333 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch apply alertmanager '1;smithi026=a' 2024-04-30T09:30:29.699 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:29 smithi046 ceph-mon[23181]: mgrmap e18: y(active, since 1.0537s), standbys: x 2024-04-30T09:30:29.699 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:29 smithi046 ceph-mon[23181]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi046=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:29.699 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:29 smithi046 ceph-mon[23181]: Saving service node-exporter spec with placement smithi026=a;smithi046=b;count:2 2024-04-30T09:30:29.699 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:29 smithi046 ceph-mon[23181]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:29.699 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:29 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:29.699 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:29 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:29.699 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:29 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:29.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[27123]: mgrmap e18: y(active, since 1.0537s), standbys: x 2024-04-30T09:30:29.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[27123]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi046=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:29.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[27123]: Saving service node-exporter spec with placement smithi026=a;smithi046=b;count:2 2024-04-30T09:30:29.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[27123]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:29.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:29.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:29.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:29.779 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[22362]: mgrmap e18: y(active, since 1.0537s), standbys: x 2024-04-30T09:30:29.780 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[22362]: from='client.24367 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi046=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:29.780 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[22362]: Saving service node-exporter spec with placement smithi026=a;smithi046=b;count:2 2024-04-30T09:30:29.780 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[22362]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:29.780 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:29.780 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:29.780 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:29 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.054 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:30 smithi046 ceph-mon[23181]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:31.055 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:30 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.055 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:30 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:30:31.055 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:30 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:30:31.055 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:30 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.055 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:30 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.055 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:30 smithi046 ceph-mon[23181]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-30T09:30:31.055 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:30 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.055 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:30 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[22362]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:31.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:30:31.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[22362]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[27123]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[27123]: mgrmap e19: y(active, since 2s), standbys: x 2024-04-30T09:30:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:30 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:31.298 INFO:teuthology.orchestra.run.smithi046.stdout:Scheduled alertmanager update... 2024-04-30T09:30:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:31 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:30:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:31 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:30:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:31 smithi046 ceph-mon[23181]: Deploying daemon prometheus.a on smithi046 2024-04-30T09:30:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:31 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:32.059 DEBUG:teuthology.orchestra.run.smithi026:alertmanager.a> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@alertmanager.a.service 2024-04-30T09:30:32.062 INFO:tasks.cephadm:Adding grafana.a on smithi046 2024-04-30T09:30:32.062 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph orch apply grafana '1;smithi046=a' 2024-04-30T09:30:32.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:31 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:30:32.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:31 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:30:32.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:31 smithi026 ceph-mon[22362]: Deploying daemon prometheus.a on smithi046 2024-04-30T09:30:32.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:31 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:32.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:31 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:30:32.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:31 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:30:32.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:31 smithi026 ceph-mon[27123]: Deploying daemon prometheus.a on smithi046 2024-04-30T09:30:32.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:31 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:32 smithi046 ceph-mon[23181]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:32 smithi046 ceph-mon[23181]: Saving service alertmanager spec with placement smithi026=a;count:1 2024-04-30T09:30:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:32 smithi046 ceph-mon[23181]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:33.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:32 smithi026 ceph-mon[27123]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:33.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:32 smithi026 ceph-mon[27123]: Saving service alertmanager spec with placement smithi026=a;count:1 2024-04-30T09:30:33.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:32 smithi026 ceph-mon[27123]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:32 smithi026 ceph-mon[22362]: from='client.24383 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:32 smithi026 ceph-mon[22362]: Saving service alertmanager spec with placement smithi026=a;count:1 2024-04-30T09:30:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:32 smithi026 ceph-mon[22362]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:34.605 INFO:teuthology.orchestra.run.smithi046.stdout:Scheduled grafana update... 2024-04-30T09:30:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:34 smithi026 ceph-mon[27123]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:34 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:34 smithi026 ceph-mon[22362]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:34 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:34 smithi046 ceph-mon[23181]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:34 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:35.785 DEBUG:teuthology.orchestra.run.smithi046:grafana.a> sudo journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@grafana.a.service 2024-04-30T09:30:35.788 INFO:tasks.cephadm:Setting up client nodes... 2024-04-30T09:30:35.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-30T09:30:36.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:35 smithi026 ceph-mon[27123]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi046=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:36.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:35 smithi026 ceph-mon[27123]: Saving service grafana spec with placement smithi046=a;count:1 2024-04-30T09:30:36.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:35 smithi026 ceph-mon[22362]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi046=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:36.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:35 smithi026 ceph-mon[22362]: Saving service grafana spec with placement smithi046=a;count:1 2024-04-30T09:30:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:35 smithi046 ceph-mon[23181]: from='client.24389 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi046=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:30:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:35 smithi046 ceph-mon[23181]: Saving service grafana spec with placement smithi046=a;count:1 2024-04-30T09:30:37.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:36 smithi026 ceph-mon[27123]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:37.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:36 smithi026 ceph-mon[22362]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:37.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:36 smithi046 ceph-mon[23181]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:38.036 INFO:teuthology.orchestra.run.smithi026.stdout:[client.0] 2024-04-30T09:30:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: key = AQA+ujBmDMXrARAAmyep52tPjX3WC1MMqOYtLA== 2024-04-30T09:30:38.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:38 smithi026 ceph-mon[27123]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:38.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:38 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3326553802' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:38.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:38 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:38.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:38 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-30T09:30:38.914 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:38 smithi026 ceph-mon[22362]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:38.914 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:38 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3326553802' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:38.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:38 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:38.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:38 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-30T09:30:38.941 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2024-04-30T09:30:38.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-30T09:30:38.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-30T09:30:38.981 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-30T09:30:39.036 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:38 smithi046 ceph-mon[23181]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:39.036 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:38 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3326553802' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:39.036 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:38 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:39.036 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:38 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-30T09:30:40.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:40 smithi046 ceph-mon[23181]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:41.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:40 smithi026 ceph-mon[22362]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:40 smithi026 ceph-mon[27123]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:42.929 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:42 smithi046 ceph-mon[23181]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:42.929 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:42 smithi046 systemd[1]: Starting Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:30:43.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:42 smithi026 ceph-mon[22362]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:42 smithi026 ceph-mon[27123]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:44.128 INFO:teuthology.orchestra.run.smithi046.stdout:[client.1] 2024-04-30T09:30:44.128 INFO:teuthology.orchestra.run.smithi046.stdout: key = AQBEujBmkgmGBhAAzyie0Nosh5UeB9ru9hNHcw== 2024-04-30T09:30:44.244 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:43 smithi046 podman[38264]: 2024-04-30 09:30:43.992063374 +0000 UTC m=+0.539008904 container create 128e33ff71d06e63c3c359124b3593b5d81f76aaf84c1424eedefd202811d7c4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 podman[38264]: 2024-04-30 09:30:43.906560217 +0000 UTC m=+0.453505755 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 podman[38264]: 2024-04-30 09:30:44.194280655 +0000 UTC m=+0.741226178 container init 128e33ff71d06e63c3c359124b3593b5d81f76aaf84c1424eedefd202811d7c4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 podman[38264]: 2024-04-30 09:30:44.1971698 +0000 UTC m=+0.744115323 container start 128e33ff71d06e63c3c359124b3593b5d81f76aaf84c1424eedefd202811d7c4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.221Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.221Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.221Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.221Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi046 (none))" 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.221Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.221Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.223Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.223Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.225Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-30T09:30:44.245 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.226Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-30T09:30:44.246 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.226Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.349µs 2024-04-30T09:30:44.246 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.226Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-30T09:30:44.246 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.226Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-30T09:30:44.246 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.226Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.377µs wal_replay_duration=120.709µs total_replay_duration=155.246µs 2024-04-30T09:30:44.246 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.227Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-30T09:30:44.246 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.227Z caller=main.go:947 level=info msg="TSDB started" 2024-04-30T09:30:44.246 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.227Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-30T09:30:44.246 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.244Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.64248ms db_storage=576ns remote_storage=1.086µs web_handler=239ns query_engine=548ns scrape=411.933µs scrape_sd=15.481µs notify=816ns notify_sd=2.351µs rules=16.065497ms 2024-04-30T09:30:44.246 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:30:44.244Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-30T09:30:44.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 bash[38264]: 128e33ff71d06e63c3c359124b3593b5d81f76aaf84c1424eedefd202811d7c4 2024-04-30T09:30:44.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:30:44 smithi046 systemd[1]: Started Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:30:44.897 DEBUG:teuthology.orchestra.run.smithi046:> set -ex 2024-04-30T09:30:44.897 DEBUG:teuthology.orchestra.run.smithi046:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-30T09:30:44.897 DEBUG:teuthology.orchestra.run.smithi046:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-30T09:30:44.936 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-30T09:30:44.937 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-30T09:30:44.937 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph mgr dump --format=json 2024-04-30T09:30:44.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[27123]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:44.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[27123]: from='client.? 172.21.15.46:0/3246583277' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:44.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:44.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-30T09:30:44.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:44.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[22362]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:44.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[22362]: from='client.? 172.21.15.46:0/3246583277' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:44.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:44.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-30T09:30:44.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:44 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-mon[23181]: pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-mon[23181]: from='client.? 172.21.15.46:0/3246583277' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-30T09:30:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-30T09:30:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:44 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:45.300 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:45.329 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:45 smithi026 systemd[1]: Starting Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:30:45.582 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:45 smithi026 bash[41983]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-30T09:30:45.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:45 smithi026 ceph-mon[27123]: Deploying daemon node-exporter.a on smithi026 2024-04-30T09:30:45.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:45 smithi026 ceph-mon[22362]: Deploying daemon node-exporter.a on smithi026 2024-04-30T09:30:45.947 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:45 smithi026 bash[41983]: Getting image source signatures 2024-04-30T09:30:45.947 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:45 smithi026 bash[41983]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-30T09:30:45.947 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:45 smithi026 bash[41983]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-30T09:30:45.947 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:45 smithi026 bash[41983]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-30T09:30:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:45 smithi046 ceph-mon[23181]: Deploying daemon node-exporter.a on smithi026 2024-04-30T09:30:47.187 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:46 smithi026 ceph-mon[27123]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:47.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:46 smithi026 ceph-mon[22362]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:46 smithi046 ceph-mon[23181]: pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:47.878 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:47 smithi026 bash[41983]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-30T09:30:48.196 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:47 smithi026 bash[41983]: Writing manifest to image destination 2024-04-30T09:30:48.384 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:30:48.557 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 podman[41983]: 2024-04-30 09:30:48.345348011 +0000 UTC m=+3.126795162 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-30T09:30:48.557 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 podman[41983]: 2024-04-30 09:30:48.452495534 +0000 UTC m=+3.233942690 container create d712368b56f8c644f5dd6568f9f6d5ecf956bb9cde0e8f5441d121c07d46a890 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:30:48.825 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-mon[27123]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:48.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:48.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2748383581' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-30T09:30:48.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-mon[22362]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:48.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:48.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2748383581' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 podman[41983]: 2024-04-30 09:30:48.655172325 +0000 UTC m=+3.436619485 container init d712368b56f8c644f5dd6568f9f6d5ecf956bb9cde0e8f5441d121c07d46a890 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 podman[41983]: 2024-04-30 09:30:48.657948658 +0000 UTC m=+3.439395808 container start d712368b56f8c644f5dd6568f9f6d5ecf956bb9cde0e8f5441d121c07d46a890 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.661Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.661Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.662Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.662Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.662Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.662Z caller=node_exporter.go:115 level=info collector=arp 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.662Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-30T09:30:48.827 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=edac 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-30T09:30:48.828 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=os 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-30T09:30:48.829 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=stat 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=time 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=uname 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[42155]: ts=2024-04-30T09:30:48.663Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 bash[41983]: d712368b56f8c644f5dd6568f9f6d5ecf956bb9cde0e8f5441d121c07d46a890 2024-04-30T09:30:48.830 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:30:48 smithi026 systemd[1]: Started Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:30:48.831 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:48 smithi046 ceph-mon[23181]: pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:48.831 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:48 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:48.831 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:48 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2748383581' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-30T09:30:49.672 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":19,"active_gid":24361,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6800","nonce":2789192078},{"type":"v1","addr":"172.21.15.26:6801","nonce":2789192078}]},"active_addr":"172.21.15.26:6801/2789192078","active_change":"2024-04-30T09:30:27.470033+0000","active_mgr_features":4540138303579357183,"available":true,"standbys":[{"gid":24344,"name":"x","mgr_features":4540138303579357183,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.23.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:8.3.5","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"docker.io/library/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"docker.io/arcts/keepalived","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.3.1","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.33.4","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"last_failure_osd_epoch":59,"active_clients":[{"addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":449056828}]},{"addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":2762166568}]},{"addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":1501562524}]},{"addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":347975072}]}]}} 2024-04-30T09:30:49.679 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-30T09:30:49.679 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-30T09:30:49.679 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-30T09:30:49.945 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:49.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:49 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:49.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:49 smithi026 ceph-mon[22362]: Deploying daemon node-exporter.b on smithi046 2024-04-30T09:30:49.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:49 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:49.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:49 smithi026 ceph-mon[27123]: Deploying daemon node-exporter.b on smithi046 2024-04-30T09:30:49.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:49 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:49.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:49 smithi046 ceph-mon[23181]: Deploying daemon node-exporter.b on smithi046 2024-04-30T09:30:49.985 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:49 smithi046 bash[38567]: Trying to pull quay.io/prometheus/node-exporter:v1.3.1... 2024-04-30T09:30:50.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:50 smithi046 bash[38567]: Getting image source signatures 2024-04-30T09:30:50.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:50 smithi046 bash[38567]: Copying blob sha256:b5db1e299295edf3005515ab7879c1df64a33c185d3a7a23aa4dcaa17d26f7b3 2024-04-30T09:30:50.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:50 smithi046 bash[38567]: Copying blob sha256:aa2a8d90b84cb2a9c422e7005cd166a008ccf22ef5d7d4f07128478585ce35ea 2024-04-30T09:30:50.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:50 smithi046 bash[38567]: Copying blob sha256:b45d31ee2d7f9f452678a85b0c837c29e12089f31ee8dbac6c8c24dfa4054a30 2024-04-30T09:30:50.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:50 smithi046 ceph-mon[23181]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:51.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:50 smithi026 ceph-mon[27123]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:51.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:50 smithi026 ceph-mon[22362]: pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:51.951 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:30:51.951 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":59,"fsid":"703f4aea-06d3-11ef-bc93-c7b262605968","created":"2024-04-30T09:25:00.900233+0000","modified":"2024-04-30T09:30:27.469676+0000","last_up_change":"2024-04-30T09:30:08.170335+0000","last_in_change":"2024-04-30T09:29:50.713788+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-30T09:28:08.974604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"031c2da1-c3a9-4d2b-83fe-93835cc5295a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6803","nonce":2102761558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6805","nonce":2102761558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6809","nonce":2102761558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6807","nonce":2102761558}]},"public_addr":"172.21.15.26:6803/2102761558","cluster_addr":"172.21.15.26:6805/2102761558","heartbeat_back_addr":"172.21.15.26:6809/2102761558","heartbeat_front_addr":"172.21.15.26:6807/2102761558","state":["exists","up"]},{"osd":1,"uuid":"59928493-d00b-4259-b638-8b4743f1434e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6811","nonce":1899832674}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6813","nonce":1899832674}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6817","nonce":1899832674}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6815","nonce":1899832674}]},"public_addr":"172.21.15.26:6811/1899832674","cluster_addr":"172.21.15.26:6813/1899832674","heartbeat_back_addr":"172.21.15.26:6817/1899832674","heartbeat_front_addr":"172.21.15.26:6815/1899832674","state":["exists","up"]},{"osd":2,"uuid":"07cad5bf-069c-4688-8de5-abcf2b56f970","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6819","nonce":3754664132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6821","nonce":3754664132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6825","nonce":3754664132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6823","nonce":3754664132}]},"public_addr":"172.21.15.26:6819/3754664132","cluster_addr":"172.21.15.26:6821/3754664132","heartbeat_back_addr":"172.21.15.26:6825/3754664132","heartbeat_front_addr":"172.21.15.26:6823/3754664132","state":["exists","up"]},{"osd":3,"uuid":"5e016838-5979-4375-aae0-8a42a81a5290","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6827","nonce":1106154924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6829","nonce":1106154924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6833","nonce":1106154924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6831","nonce":1106154924}]},"public_addr":"172.21.15.26:6827/1106154924","cluster_addr":"172.21.15.26:6829/1106154924","heartbeat_back_addr":"172.21.15.26:6833/1106154924","heartbeat_front_addr":"172.21.15.26:6831/1106154924","state":["exists","up"]},{"osd":4,"uuid":"c4ea38ab-de25-4ee8-a55c-e75a516ff355","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6800","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6801","nonce":3142607282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6802","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6803","nonce":3142607282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6806","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6807","nonce":3142607282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6804","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6805","nonce":3142607282}]},"public_addr":"172.21.15.46:6801/3142607282","cluster_addr":"172.21.15.46:6803/3142607282","heartbeat_back_addr":"172.21.15.46:6807/3142607282","heartbeat_front_addr":"172.21.15.46:6805/3142607282","state":["exists","up"]},{"osd":5,"uuid":"6bcfd398-fdfe-4be8-b5ed-da02488b4a3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6808","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6809","nonce":2766677176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6810","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6811","nonce":2766677176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6814","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6815","nonce":2766677176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6812","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6813","nonce":2766677176}]},"public_addr":"172.21.15.46:6809/2766677176","cluster_addr":"172.21.15.46:6811/2766677176","heartbeat_back_addr":"172.21.15.46:6815/2766677176","heartbeat_front_addr":"172.21.15.46:6813/2766677176","state":["exists","up"]},{"osd":6,"uuid":"a2773219-a6c2-414e-b2d0-f381f881c4af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6816","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6817","nonce":703785050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6818","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6819","nonce":703785050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6822","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6823","nonce":703785050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6820","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6821","nonce":703785050}]},"public_addr":"172.21.15.46:6817/703785050","cluster_addr":"172.21.15.46:6819/703785050","heartbeat_back_addr":"172.21.15.46:6823/703785050","heartbeat_front_addr":"172.21.15.46:6821/703785050","state":["exists","up"]},{"osd":7,"uuid":"04ed7db9-9f70-4d69-8e14-2f738d07bfdd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6824","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6825","nonce":4197915406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6826","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6827","nonce":4197915406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6830","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6831","nonce":4197915406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6828","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6829","nonce":4197915406}]},"public_addr":"172.21.15.46:6825/4197915406","cluster_addr":"172.21.15.46:6827/4197915406","heartbeat_back_addr":"172.21.15.46:6831/4197915406","heartbeat_front_addr":"172.21.15.46:6829/4197915406","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:27:16.963970+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:27:40.868367+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:28:05.980512+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:28:31.450304+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:28:53.337778+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:29:17.234061+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:29:41.061817+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:30:06.161935+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/209764474":"2024-05-01T09:30:27.469621+0000","172.21.15.26:0/753133130":"2024-05-01T09:30:27.469621+0000","172.21.15.26:0/3645261121":"2024-05-01T09:30:27.469621+0000","172.21.15.26:6800/2396108168":"2024-05-01T09:30:27.469621+0000","172.21.15.26:0/3683337897":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/239085219":"2024-05-01T09:25:55.470596+0000","172.21.15.26:6801/1844994599":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/3072786577":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/514779481":"2024-05-01T09:25:32.971742+0000","172.21.15.26:0/326280148":"2024-05-01T09:30:27.469621+0000","172.21.15.26:0/2113986196":"2024-05-01T09:25:32.971742+0000","172.21.15.26:6801/733429943":"2024-05-01T09:25:32.971742+0000","172.21.15.26:6800/733429943":"2024-05-01T09:25:32.971742+0000","172.21.15.26:6801/2396108168":"2024-05-01T09:30:27.469621+0000","172.21.15.26:6800/1844994599":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/4095183837":"2024-05-01T09:25:32.971742+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-30T09:30:52.181 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:51 smithi046 bash[38567]: Copying config sha256:1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 2024-04-30T09:30:52.181 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:51 smithi046 bash[38567]: Writing manifest to image destination 2024-04-30T09:30:52.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 podman[38567]: 2024-04-30 09:30:52.109275648 +0000 UTC m=+2.353351930 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 podman[38567]: 2024-04-30 09:30:52.216619921 +0000 UTC m=+2.460696185 container create 727cc4da58b386ae792dd1338a37247198bf495d31bb36dae6fb422d58093ae5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 podman[38567]: 2024-04-30 09:30:52.401977079 +0000 UTC m=+2.646053344 container init 727cc4da58b386ae792dd1338a37247198bf495d31bb36dae6fb422d58093ae5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 podman[38567]: 2024-04-30 09:30:52.404818391 +0000 UTC m=+2.648894657 container start 727cc4da58b386ae792dd1338a37247198bf495d31bb36dae6fb422d58093ae5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.409Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.409Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.409Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.409Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=arp 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-30T09:30:52.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=edac 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-30T09:30:52.486 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=os 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-30T09:30:52.487 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=stat 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=time 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=uname 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-30T09:30:52.488 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-30T09:30:52.489 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.410Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-30T09:30:52.489 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[38617]: ts=2024-04-30T09:30:52.411Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-30T09:30:52.489 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 bash[38567]: 727cc4da58b386ae792dd1338a37247198bf495d31bb36dae6fb422d58093ae5 2024-04-30T09:30:52.579 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:30:52] "GET /metrics HTTP/1.1" 200 191489 "" "Prometheus/2.33.4" 2024-04-30T09:30:52.580 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-30T09:30:52.581 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd dump --format=json 2024-04-30T09:30:52.803 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:52.808 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:30:52 smithi046 systemd[1]: Started Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:30:52.928 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[27123]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:52.928 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1281509981' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[22362]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1281509981' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:52.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:52 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:53.074 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-mon[23181]: pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:53.074 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1281509981' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-30T09:30:53.074 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:53.074 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:30:53.075 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:30:53.075 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:53.075 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:52 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:30:54.598 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:30:54.598 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":59,"fsid":"703f4aea-06d3-11ef-bc93-c7b262605968","created":"2024-04-30T09:25:00.900233+0000","modified":"2024-04-30T09:30:27.469676+0000","last_up_change":"2024-04-30T09:30:08.170335+0000","last_in_change":"2024-04-30T09:29:50.713788+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-30T09:28:08.974604+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"031c2da1-c3a9-4d2b-83fe-93835cc5295a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6803","nonce":2102761558}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6805","nonce":2102761558}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6809","nonce":2102761558}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":2102761558},{"type":"v1","addr":"172.21.15.26:6807","nonce":2102761558}]},"public_addr":"172.21.15.26:6803/2102761558","cluster_addr":"172.21.15.26:6805/2102761558","heartbeat_back_addr":"172.21.15.26:6809/2102761558","heartbeat_front_addr":"172.21.15.26:6807/2102761558","state":["exists","up"]},{"osd":1,"uuid":"59928493-d00b-4259-b638-8b4743f1434e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6811","nonce":1899832674}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6813","nonce":1899832674}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6817","nonce":1899832674}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":1899832674},{"type":"v1","addr":"172.21.15.26:6815","nonce":1899832674}]},"public_addr":"172.21.15.26:6811/1899832674","cluster_addr":"172.21.15.26:6813/1899832674","heartbeat_back_addr":"172.21.15.26:6817/1899832674","heartbeat_front_addr":"172.21.15.26:6815/1899832674","state":["exists","up"]},{"osd":2,"uuid":"07cad5bf-069c-4688-8de5-abcf2b56f970","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6819","nonce":3754664132}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6821","nonce":3754664132}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6825","nonce":3754664132}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3754664132},{"type":"v1","addr":"172.21.15.26:6823","nonce":3754664132}]},"public_addr":"172.21.15.26:6819/3754664132","cluster_addr":"172.21.15.26:6821/3754664132","heartbeat_back_addr":"172.21.15.26:6825/3754664132","heartbeat_front_addr":"172.21.15.26:6823/3754664132","state":["exists","up"]},{"osd":3,"uuid":"5e016838-5979-4375-aae0-8a42a81a5290","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6827","nonce":1106154924}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6829","nonce":1106154924}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6833","nonce":1106154924}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":1106154924},{"type":"v1","addr":"172.21.15.26:6831","nonce":1106154924}]},"public_addr":"172.21.15.26:6827/1106154924","cluster_addr":"172.21.15.26:6829/1106154924","heartbeat_back_addr":"172.21.15.26:6833/1106154924","heartbeat_front_addr":"172.21.15.26:6831/1106154924","state":["exists","up"]},{"osd":4,"uuid":"c4ea38ab-de25-4ee8-a55c-e75a516ff355","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6800","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6801","nonce":3142607282}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6802","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6803","nonce":3142607282}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6806","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6807","nonce":3142607282}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6804","nonce":3142607282},{"type":"v1","addr":"172.21.15.46:6805","nonce":3142607282}]},"public_addr":"172.21.15.46:6801/3142607282","cluster_addr":"172.21.15.46:6803/3142607282","heartbeat_back_addr":"172.21.15.46:6807/3142607282","heartbeat_front_addr":"172.21.15.46:6805/3142607282","state":["exists","up"]},{"osd":5,"uuid":"6bcfd398-fdfe-4be8-b5ed-da02488b4a3c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6808","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6809","nonce":2766677176}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6810","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6811","nonce":2766677176}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6814","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6815","nonce":2766677176}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6812","nonce":2766677176},{"type":"v1","addr":"172.21.15.46:6813","nonce":2766677176}]},"public_addr":"172.21.15.46:6809/2766677176","cluster_addr":"172.21.15.46:6811/2766677176","heartbeat_back_addr":"172.21.15.46:6815/2766677176","heartbeat_front_addr":"172.21.15.46:6813/2766677176","state":["exists","up"]},{"osd":6,"uuid":"a2773219-a6c2-414e-b2d0-f381f881c4af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6816","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6817","nonce":703785050}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6818","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6819","nonce":703785050}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6822","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6823","nonce":703785050}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6820","nonce":703785050},{"type":"v1","addr":"172.21.15.46:6821","nonce":703785050}]},"public_addr":"172.21.15.46:6817/703785050","cluster_addr":"172.21.15.46:6819/703785050","heartbeat_back_addr":"172.21.15.46:6823/703785050","heartbeat_front_addr":"172.21.15.46:6821/703785050","state":["exists","up"]},{"osd":7,"uuid":"04ed7db9-9f70-4d69-8e14-2f738d07bfdd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6824","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6825","nonce":4197915406}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6826","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6827","nonce":4197915406}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6830","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6831","nonce":4197915406}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.46:6828","nonce":4197915406},{"type":"v1","addr":"172.21.15.46:6829","nonce":4197915406}]},"public_addr":"172.21.15.46:6825/4197915406","cluster_addr":"172.21.15.46:6827/4197915406","heartbeat_back_addr":"172.21.15.46:6831/4197915406","heartbeat_front_addr":"172.21.15.46:6829/4197915406","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:27:16.963970+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:27:40.868367+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:28:05.980512+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:28:31.450304+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:28:53.337778+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:29:17.234061+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:29:41.061817+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138303579357183,"old_weight":0,"last_purged_snaps_scrub":"2024-04-30T09:30:06.161935+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/209764474":"2024-05-01T09:30:27.469621+0000","172.21.15.26:0/753133130":"2024-05-01T09:30:27.469621+0000","172.21.15.26:0/3645261121":"2024-05-01T09:30:27.469621+0000","172.21.15.26:6800/2396108168":"2024-05-01T09:30:27.469621+0000","172.21.15.26:0/3683337897":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/239085219":"2024-05-01T09:25:55.470596+0000","172.21.15.26:6801/1844994599":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/3072786577":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/514779481":"2024-05-01T09:25:32.971742+0000","172.21.15.26:0/326280148":"2024-05-01T09:30:27.469621+0000","172.21.15.26:0/2113986196":"2024-05-01T09:25:32.971742+0000","172.21.15.26:6801/733429943":"2024-05-01T09:25:32.971742+0000","172.21.15.26:6800/733429943":"2024-05-01T09:25:32.971742+0000","172.21.15.26:6801/2396108168":"2024-05-01T09:30:27.469621+0000","172.21.15.26:6800/1844994599":"2024-05-01T09:25:55.470596+0000","172.21.15.26:0/4095183837":"2024-05-01T09:25:32.971742+0000"},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-30T09:30:54.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:54 smithi026 ceph-mon[27123]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:54.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:54 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3295541816' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-30T09:30:54.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:54 smithi026 ceph-mon[22362]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:54.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:54 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3295541816' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-30T09:30:54.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:54 smithi046 ceph-mon[23181]: pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:54.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:54 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3295541816' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-30T09:30:55.325 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-30T09:30:55.326 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-30T09:30:55.326 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-30T09:30:55.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-30T09:30:55.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-30T09:30:55.328 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-30T09:30:55.328 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-30T09:30:55.328 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-30T09:30:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:30:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:30:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:30:55.807 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:55.807 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:55.808 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:55.808 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:55.817 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:55.822 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:55.823 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:55.855 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:30:56.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:56 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:56.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:56 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:56.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:56 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:30:57.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:57 smithi026 ceph-mon[27123]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:57.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:57 smithi026 ceph-mon[22362]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:57 smithi046 ceph-mon[23181]: pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:30:59 smithi026 ceph-mon[27123]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:59.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:30:59 smithi026 ceph-mon[22362]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:30:59.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:30:59 smithi046 ceph-mon[23181]: pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:01.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:01 smithi026 ceph-mon[27123]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:01.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:01 smithi026 ceph-mon[22362]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:01 smithi046 ceph-mon[23181]: pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:02.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:31:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:02] "GET /metrics HTTP/1.1" 200 191482 "" "Prometheus/2.33.4" 2024-04-30T09:31:03.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:03 smithi026 ceph-mon[27123]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:03.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:03 smithi026 ceph-mon[22362]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:03.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:03 smithi046 ceph-mon[23181]: pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:05.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:05 smithi026 ceph-mon[27123]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:05 smithi026 ceph-mon[22362]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:05 smithi046 ceph-mon[23181]: pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:31:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:31:06.758 INFO:teuthology.orchestra.run.smithi026.stdout:231928233996 2024-04-30T09:31:06.758 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-30T09:31:07.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:07 smithi026 ceph-mon[27123]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:07.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:07 smithi026 ceph-mon[22362]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:07 smithi046 ceph-mon[23181]: pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:07.865 INFO:teuthology.orchestra.run.smithi026.stdout:128849018908 2024-04-30T09:31:07.865 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-30T09:31:08.838 INFO:teuthology.orchestra.run.smithi026.stdout:197568495634 2024-04-30T09:31:08.839 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-30T09:31:09.357 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:09 smithi026 ceph-mon[27123]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:09.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:09 smithi026 ceph-mon[22362]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:09.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:09 smithi046 ceph-mon[23181]: pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:11.187 INFO:teuthology.orchestra.run.smithi026.stdout:163208757272 2024-04-30T09:31:11.187 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-30T09:31:11.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:11 smithi026 ceph-mon[27123]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:11.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:11 smithi026 ceph-mon[22362]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:11 smithi046 ceph-mon[23181]: pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:11.943 INFO:teuthology.orchestra.run.smithi026.stdout:34359738416 2024-04-30T09:31:11.943 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-30T09:31:12.270 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:12.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:31:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:12] "GET /metrics HTTP/1.1" 200 191482 "" "Prometheus/2.33.4" 2024-04-30T09:31:12.750 INFO:teuthology.orchestra.run.smithi026.stdout:107374182432 2024-04-30T09:31:12.751 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-30T09:31:13.084 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:13.086 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:13.086 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:13.417 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:13 smithi026 ceph-mon[27123]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:13.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:13 smithi026 ceph-mon[22362]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:13 smithi046 ceph-mon[23181]: pgmap v25: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:14 smithi026 ceph-mon[27123]: Deploying daemon alertmanager.a on smithi026 2024-04-30T09:31:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:14 smithi026 ceph-mon[22362]: Deploying daemon alertmanager.a on smithi026 2024-04-30T09:31:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:14 smithi046 ceph-mon[23181]: Deploying daemon alertmanager.a on smithi026 2024-04-30T09:31:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:15 smithi026 ceph-mon[27123]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:15 smithi026 ceph-mon[22362]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:15 smithi046 ceph-mon[23181]: pgmap v26: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:31:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:31:17.126 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:17.151 INFO:teuthology.orchestra.run.smithi026.stdout:77309411367 2024-04-30T09:31:17.151 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-30T09:31:17.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:17 smithi026 ceph-mon[27123]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:17.414 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:17 smithi026 ceph-mon[22362]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:17.419 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:17.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:17 smithi046 ceph-mon[23181]: pgmap v27: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:19.406 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:19 smithi026 ceph-mon[27123]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:19.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:19 smithi026 ceph-mon[22362]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:19 smithi046 ceph-mon[23181]: pgmap v28: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:21.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:21 smithi026 ceph-mon[27123]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:21.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:21 smithi026 ceph-mon[22362]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:21.456 INFO:teuthology.orchestra.run.smithi026.stdout:128849018911 2024-04-30T09:31:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:21 smithi046 ceph-mon[23181]: pgmap v29: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:22.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:22 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3769669459' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-30T09:31:22.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:22 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1249442748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-30T09:31:22.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:22 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3769669459' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-30T09:31:22.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:22 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1249442748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-30T09:31:22.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:31:22 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:22] "GET /metrics HTTP/1.1" 200 191463 "" "Prometheus/2.33.4" 2024-04-30T09:31:22.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:22 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3769669459' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-30T09:31:22.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:22 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1249442748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-30T09:31:23.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:23 smithi026 ceph-mon[27123]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:23 smithi026 ceph-mon[22362]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:23.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:23 smithi046 ceph-mon[23181]: pgmap v30: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:23.761 INFO:teuthology.orchestra.run.smithi026.stdout:197568495637 2024-04-30T09:31:24.365 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:24 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/347772693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-30T09:31:24.366 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:24 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/347772693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-30T09:31:24.389 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495634 got 197568495637 for osd.6 2024-04-30T09:31:24.389 DEBUG:teuthology.parallel:result is None 2024-04-30T09:31:24.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:24 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/347772693' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-30T09:31:25.199 INFO:teuthology.orchestra.run.smithi026.stdout:231928234000 2024-04-30T09:31:25.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:25 smithi026 ceph-mon[27123]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:25.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:25 smithi026 ceph-mon[22362]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:25.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:25 smithi046 ceph-mon[23181]: pgmap v31: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:31:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:31:26.438 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:26 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3392320629' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-30T09:31:26.439 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:26 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3392320629' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-30T09:31:26.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:26 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3392320629' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-30T09:31:26.512 INFO:teuthology.orchestra.run.smithi026.stdout:163208757275 2024-04-30T09:31:27.075 INFO:teuthology.orchestra.run.smithi026.stdout:107374182436 2024-04-30T09:31:27.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:27 smithi026 ceph-mon[27123]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:27.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:27 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3460000173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-30T09:31:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:27 smithi026 ceph-mon[22362]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:27 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3460000173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-30T09:31:27.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:27 smithi046 ceph-mon[23181]: pgmap v32: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:27 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3460000173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-30T09:31:28.122 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757272 got 163208757275 for osd.5 2024-04-30T09:31:28.122 DEBUG:teuthology.parallel:result is None 2024-04-30T09:31:28.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:31:28.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:31:28.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:31:28.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:31:28.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:31:28.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:31:28.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:31:28.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:31:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:31:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:31:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:31:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:31:29.045 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:28 smithi026 systemd[1]: Starting Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:31:29.074 INFO:teuthology.orchestra.run.smithi026.stdout:55834574894 2024-04-30T09:31:29.074 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-30T09:31:29.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:29 smithi026 ceph-mon[27123]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:29.414 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:29 smithi026 ceph-mon[22362]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:29 smithi046 ceph-mon[23181]: pgmap v33: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:29.917 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:29.945 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233996 got 231928234000 for osd.7 2024-04-30T09:31:29.945 DEBUG:teuthology.parallel:result is None 2024-04-30T09:31:30.590 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018908 got 128849018911 for osd.4 2024-04-30T09:31:30.590 DEBUG:teuthology.parallel:result is None 2024-04-30T09:31:31.175 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:31 smithi026 podman[45084]: 2024-04-30 09:31:31.018016394 +0000 UTC m=+0.200695719 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-30T09:31:31.198 INFO:teuthology.orchestra.run.smithi026.stdout:34359738420 2024-04-30T09:31:31.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:31 smithi026 ceph-mon[27123]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:31.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:31 smithi026 ceph-mon[22362]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:31 smithi046 ceph-mon[23181]: pgmap v34: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:31.828 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:31 smithi026 podman[45084]: 2024-04-30 09:31:31.570101973 +0000 UTC m=+0.752781327 volume create 5f5134ec4070d815ecd8de8e0ccf1be866c1566d5940c74554df06a50274dc21 2024-04-30T09:31:31.828 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:31 smithi026 podman[45084]: 2024-04-30 09:31:31.66235612 +0000 UTC m=+0.845035477 container create 354b3c369536731fd7173078d750c61a87d50cb80b8a837fde391c059e2bb2d4 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:31:32.169 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182432 got 107374182436 for osd.3 2024-04-30T09:31:32.169 DEBUG:teuthology.parallel:result is None 2024-04-30T09:31:32.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1055841439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-30T09:31:32.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1055841439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-30T09:31:32.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:32] "GET /metrics HTTP/1.1" 200 191465 "" "Prometheus/2.33.4" 2024-04-30T09:31:32.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:32 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1055841439' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-30T09:31:32.765 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738420 for osd.0 2024-04-30T09:31:32.765 DEBUG:teuthology.parallel:result is None 2024-04-30T09:31:32.999 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 podman[45084]: 2024-04-30 09:31:32.870264149 +0000 UTC m=+2.052943461 container init 354b3c369536731fd7173078d750c61a87d50cb80b8a837fde391c059e2bb2d4 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:31:32.999 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 podman[45084]: 2024-04-30 09:31:32.873780982 +0000 UTC m=+2.056460295 container start 354b3c369536731fd7173078d750c61a87d50cb80b8a837fde391c059e2bb2d4 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:31:32.999 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:32.887Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-30T09:31:32.999 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:32.887Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-30T09:31:32.999 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:32.889Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-04-30T09:31:32.999 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:32.890Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-30T09:31:32.999 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:32.918Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-30T09:31:32.999 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:32.918Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-30T09:31:32.999 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:32.920Z caller=main.go:518 msg=Listening address=:9093 2024-04-30T09:31:33.000 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:32.920Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-30T09:31:33.000 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 bash[45084]: 354b3c369536731fd7173078d750c61a87d50cb80b8a837fde391c059e2bb2d4 2024-04-30T09:31:33.000 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:32 smithi026 systemd[1]: Started Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:31:33.033 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:33.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[27123]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:33.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:31:33.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[22362]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:33.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:31:33.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:33 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:33 smithi046 ceph-mon[23181]: pgmap v35: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:33 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:33 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:33 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:33 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:31:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:33 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:34.097 INFO:teuthology.orchestra.run.smithi026.stdout:77309411371 2024-04-30T09:31:34.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:34 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:31:34.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:34 smithi026 ceph-mon[27123]: Deploying daemon grafana.a on smithi046 2024-04-30T09:31:34.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:34 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1481446819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-30T09:31:34.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:34 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:31:34.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:34 smithi026 ceph-mon[22362]: Deploying daemon grafana.a on smithi046 2024-04-30T09:31:34.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:34 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1481446819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-30T09:31:34.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:34 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:31:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:34 smithi046 ceph-mon[23181]: Deploying daemon grafana.a on smithi046 2024-04-30T09:31:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:34 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1481446819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-30T09:31:35.127 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:34.890Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000168824s 2024-04-30T09:31:35.150 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411367 got 77309411371 for osd.2 2024-04-30T09:31:35.150 DEBUG:teuthology.parallel:result is None 2024-04-30T09:31:35.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:35 smithi026 ceph-mon[22362]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:35.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:35 smithi026 ceph-mon[27123]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:35.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:35 smithi046 ceph-mon[23181]: pgmap v36: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:31:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:31:36.274 INFO:teuthology.orchestra.run.smithi026.stdout:55834574897 2024-04-30T09:31:36.842 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574894 got 55834574897 for osd.1 2024-04-30T09:31:36.842 DEBUG:teuthology.parallel:result is None 2024-04-30T09:31:36.842 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-30T09:31:36.843 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-30T09:31:37.070 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:37 smithi026 ceph-mon[27123]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:37 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1334248956' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-30T09:31:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:37 smithi026 ceph-mon[22362]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:37 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1334248956' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-30T09:31:37.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:37 smithi046 ceph-mon[23181]: pgmap v37: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:37.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:37 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1334248956' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-30T09:31:38.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:38 smithi026 ceph-mon[27123]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:38 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:38.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:38 smithi026 ceph-mon[22362]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:38.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:38 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:38.941 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:31:38.942 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2024-04-30T09:31:38.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:38 smithi046 ceph-mon[23181]: pgmap v38: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:38 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:39.568 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":38,"stamp":"2024-04-30T09:31:37.527510+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48904,"kb_used_data":6088,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912312,"statfs":{"total":767960285184,"available":767910207488,"internally_reserved":0,"allocated":6234112,"data_stored":3488394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002181"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-30T09:30:27.594770+0000","last_change":"2024-04-30T09:30:11.525607+0000","last_active":"2024-04-30T09:30:27.594770+0000","last_peered":"2024-04-30T09:30:27.594770+0000","last_clean":"2024-04-30T09:30:27.594770+0000","last_became_active":"2024-04-30T09:30:11.525075+0000","last_became_peered":"2024-04-30T09:30:11.525075+0000","last_unstale":"2024-04-30T09:30:27.594770+0000","last_undegraded":"2024-04-30T09:30:27.594770+0000","last_fullsized":"2024-04-30T09:30:27.594770+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-30T09:28:08.975123+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-30T09:28:08.975123+0000","last_clean_scrub_stamp":"2024-04-30T09:28:08.975123+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-01T09:36:23.949325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.65600000000000003}]},{"osd":1,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.72799999999999998}]},{"osd":2,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.45400000000000001}]},{"osd":3,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.68000000000000005}]},{"osd":4,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.52100000000000002}]},{"osd":5,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.70099999999999996}]},{"osd":6,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.66900000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495640,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.59999999999999998}]},{"osd":1,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.47699999999999998}]},{"osd":2,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.64800000000000002}]},{"osd":3,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.77,"5min":2.77,"15min":2.77},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":2.843,"5min":2.843,"15min":2.843},"last":0.63800000000000001}]},{"osd":4,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.68500000000000005}]},{"osd":5,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.505},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.624}]},{"osd":7,"last update":"Tue Apr 30 09:31:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.61599999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6336,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:30:48 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.36599999999999999,"15min":0.34200000000000003},"min":{"1min":0.26000000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.38},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.38900000000000001,"15min":0.36299999999999999},"min":{"1min":0.29599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.60599999999999998,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.434}]},{"osd":2,"last update":"Tue Apr 30 09:31:13 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.42199999999999999,"15min":0.40799999999999997},"min":{"1min":0.315,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.59299999999999997,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39700000000000002,"15min":0.39000000000000001},"min":{"1min":0.27200000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.54300000000000004,"5min":0.622,"15min":0.622},"last":0.34599999999999997}]},{"osd":3,"last update":"Tue Apr 30 09:30:34 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.48999999999999999,"15min":0.48799999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.65100000000000002,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.36299999999999999}]},{"osd":4,"last update":"Tue Apr 30 09:31:00 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.64300000000000002,"15min":0.64500000000000002},"min":{"1min":0.45900000000000002,"5min":0.439,"15min":0.439},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64100000000000001,"15min":0.64600000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.83099999999999996,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Tue Apr 30 09:31:27 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.65100000000000002,"15min":0.65300000000000002},"min":{"1min":0.42799999999999999,"5min":0.375,"15min":0.375},"max":{"1min":0.872,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.628},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65200000000000002,"15min":0.65100000000000002},"min":{"1min":0.48599999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.74199999999999999}]},{"osd":6,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.70199999999999996}]},{"osd":7,"last update":"Tue Apr 30 09:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.73199999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6916,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738236,"statfs":{"total":95995035648,"available":95987953664,"internally_reserved":0,"allocated":1110016,"data_stored":766504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 30 09:30:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.36499999999999999,"15min":0.34699999999999998},"min":{"1min":0.23200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.38900000000000001,"15min":0.36299999999999999},"min":{"1min":0.28100000000000003,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.28000000000000003}]},{"osd":2,"last update":"Tue Apr 30 09:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.42099999999999999,"15min":0.40999999999999998},"min":{"1min":0.33600000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.41899999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.58399999999999996,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.371}]},{"osd":3,"last update":"Tue Apr 30 09:30:38 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.47699999999999998,"15min":0.47899999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.58199999999999996,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.47699999999999998,"15min":0.47599999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.64400000000000002,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.29599999999999999}]},{"osd":4,"last update":"Tue Apr 30 09:30:57 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.56899999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.77500000000000002,"5min":0.872,"15min":0.872},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.623,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.40799999999999997,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.64400000000000002}]},{"osd":5,"last update":"Tue Apr 30 09:31:27 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.621,"15min":0.623},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.80600000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.59099999999999997,"15min":0.59299999999999997},"min":{"1min":0.41199999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.70799999999999996,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.82699999999999996}]},{"osd":6,"last update":"Tue Apr 30 09:30:45 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.63500000000000001}]},{"osd":7,"last update":"Tue Apr 30 09:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.66800000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.377,"15min":0.36099999999999999},"min":{"1min":0.27500000000000002,"5min":0.193,"15min":0.193},"max":{"1min":0.79400000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.38},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.40100000000000002,"15min":0.379},"min":{"1min":0.26900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.41499999999999998}]},{"osd":1,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41099999999999998,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.82899999999999996,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.438,"5min":0.41699999999999998,"15min":0.39700000000000002},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.69499999999999995,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.35299999999999998}]},{"osd":3,"last update":"Tue Apr 30 09:30:37 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.48899999999999999,"15min":0.49099999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.61899999999999999,"5min":1.044,"15min":1.044},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.495,"15min":0.495},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.47399999999999998}]},{"osd":4,"last update":"Tue Apr 30 09:31:01 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.627,"15min":0.626},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.63800000000000001,"15min":0.63700000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.871,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.504}]},{"osd":5,"last update":"Tue Apr 30 09:31:21 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.68000000000000005,"15min":0.68200000000000005},"min":{"1min":0.48899999999999999,"5min":0.44,"15min":0.44},"max":{"1min":1.1160000000000001,"5min":1.1160000000000001,"15min":1.1160000000000001},"last":0.56699999999999995}]},{"osd":6,"last update":"Tue Apr 30 09:30:45 2024","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.497},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.59999999999999998}]},{"osd":7,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.0920000000000001,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.64900000000000002}]}]},{"osd":3,"up_from":25,"seq":107374182438,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.41899999999999998,"15min":0.41599999999999998},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.439,"15min":0.44800000000000001},"min":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.61299999999999999,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.35499999999999998}]},{"osd":1,"last update":"Tue Apr 30 09:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.441},{"interface":"front","average":{"1min":0.439,"5min":0.45500000000000002,"15min":0.45800000000000002},"min":{"1min":0.32700000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.60299999999999998,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.378}]},{"osd":2,"last update":"Tue Apr 30 09:30:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.48899999999999999,"15min":0.49299999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.435,"5min":0.47899999999999998,"15min":0.48599999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.39200000000000002}]},{"osd":4,"last update":"Tue Apr 30 09:30:57 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.64100000000000001,"15min":0.64400000000000002},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.92700000000000005,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63500000000000001,"15min":0.63400000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.92600000000000005,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.51000000000000001}]},{"osd":5,"last update":"Tue Apr 30 09:31:22 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.66500000000000004,"15min":0.67300000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.88800000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.68200000000000005,"15min":0.69299999999999995},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.97699999999999998,"5min":1.155,"15min":1.155},"last":0.47299999999999998}]},{"osd":6,"last update":"Tue Apr 30 09:30:46 2024","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.60299999999999998}]},{"osd":7,"last update":"Tue Apr 30 09:31:10 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.58299999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018914,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.61399999999999999,"15min":0.63100000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.74299999999999999,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.60099999999999998,"15min":0.61599999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.67800000000000005}]},{"osd":1,"last update":"Tue Apr 30 09:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.57699999999999996,"15min":0.58199999999999996},"min":{"1min":0.41399999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.78000000000000003,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.625,"15min":0.63900000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.72099999999999997,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.59999999999999998}]},{"osd":2,"last update":"Tue Apr 30 09:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.59899999999999998,"15min":0.61099999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.61099999999999999,"15min":0.627},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.75800000000000001,"5min":0.88,"15min":0.88},"last":0.70499999999999996}]},{"osd":3,"last update":"Tue Apr 30 09:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.59499999999999997,"15min":0.60799999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.70299999999999996,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.63800000000000001,"15min":0.65400000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69799999999999995,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.66300000000000003}]},{"osd":5,"last update":"Tue Apr 30 09:31:22 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.439,"15min":0.432},"min":{"1min":0.35699999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.42799999999999999,"15min":0.41699999999999998},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.313},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.41399999999999998}]},{"osd":6,"last update":"Tue Apr 30 09:30:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.441},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.46400000000000002}]},{"osd":7,"last update":"Tue Apr 30 09:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.495},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.47699999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757278,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5684,"kb_used_data":556,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739468,"statfs":{"total":95995035648,"available":95989215232,"internally_reserved":0,"allocated":569344,"data_stored":236988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.62,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.66800000000000004}]},{"osd":1,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.57199999999999995,"15min":0.57999999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.73799999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.69499999999999995}]},{"osd":2,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.75800000000000001,"5min":0.999,"15min":0.999},"last":0.505},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.56200000000000006,"15min":0.56699999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.67100000000000004,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.622}]},{"osd":3,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56799999999999995,"15min":0.57199999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.752},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.56499999999999995,"15min":0.56899999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72299999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.68300000000000005}]},{"osd":4,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.42099999999999999,"15min":0.41199999999999998},"min":{"1min":0.28899999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.44},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.46000000000000002,"15min":0.45800000000000002},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.65700000000000003}]},{"osd":6,"last update":"Tue Apr 30 09:30:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.77700000000000002}]},{"osd":7,"last update":"Tue Apr 30 09:31:14 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.59899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-30T09:31:39.570 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph pg dump --format=json 2024-04-30T09:31:39.792 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:39.884 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:39 smithi046 ceph-mon[23181]: from='client.14637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-30T09:31:39.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:39 smithi026 ceph-mon[27123]: from='client.14637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-30T09:31:39.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:39 smithi026 ceph-mon[22362]: from='client.14637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-30T09:31:40.898 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:40 smithi046 ceph-mon[23181]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:40.907 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:40 smithi026 ceph-mon[27123]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:40.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:40 smithi026 ceph-mon[22362]: pgmap v39: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:41.602 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:31:41.603 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2024-04-30T09:31:42.254 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":40,"stamp":"2024-04-30T09:31:41.528113+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":3,"num_osds":8,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":749961216,"kb_used":48904,"kb_used_data":6088,"kb_used_omap":0,"kb_used_meta":42752,"kb_avail":749912312,"statfs":{"total":767960285184,"available":767910207488,"internally_reserved":0,"allocated":6234112,"data_stored":3488394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":43778048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001924"},"pg_stats":[{"pgid":"1.0","version":"59'128","reported_seq":267,"reported_epoch":59,"state":"active+clean","last_fresh":"2024-04-30T09:30:27.594770+0000","last_change":"2024-04-30T09:30:11.525607+0000","last_active":"2024-04-30T09:30:27.594770+0000","last_peered":"2024-04-30T09:30:27.594770+0000","last_clean":"2024-04-30T09:30:27.594770+0000","last_became_active":"2024-04-30T09:30:11.525075+0000","last_became_peered":"2024-04-30T09:30:11.525075+0000","last_unstale":"2024-04-30T09:30:27.594770+0000","last_undegraded":"2024-04-30T09:30:27.594770+0000","last_fullsized":"2024-04-30T09:30:27.594770+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-30T09:28:08.975123+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-30T09:28:08.975123+0000","last_clean_scrub_stamp":"2024-04-30T09:28:08.975123+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-05-01T09:36:23.949325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1586784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1597440,"data_stored":1586784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234004,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6088,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":4992,"kb_avail":93739064,"statfs":{"total":95995035648,"available":95988801536,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5111808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.60199999999999998}]},{"osd":1,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.504,"15min":0.504},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.53200000000000003}]},{"osd":2,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.49299999999999999}]},{"osd":3,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.433},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.42199999999999999}]},{"osd":4,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.51600000000000001}]},{"osd":5,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.64600000000000002}]},{"osd":6,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.627},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.56399999999999995}]}]},{"osd":6,"up_from":46,"seq":197568495641,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6152,"kb_used_data":1088,"kb_used_omap":0,"kb_used_meta":5056,"kb_avail":93739000,"statfs":{"total":95995035648,"available":95988736000,"internally_reserved":0,"allocated":1114112,"data_stored":766769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5177344},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.41799999999999998}]},{"osd":1,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.432}]},{"osd":2,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.69999999999999996,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.38800000000000001}]},{"osd":3,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.77,"5min":2.77,"15min":2.77},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":2.843,"5min":2.843,"15min":2.843},"last":0.48599999999999999}]},{"osd":4,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.81999999999999995,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.315},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.29899999999999999}]},{"osd":5,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.39900000000000002}]},{"osd":7,"last update":"Tue Apr 30 09:31:10 2024","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.46400000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6336,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:30:48 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.36599999999999999,"15min":0.34200000000000003},"min":{"1min":0.26000000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.38900000000000001,"15min":0.36299999999999999},"min":{"1min":0.29599999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.60599999999999998,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.59699999999999998}]},{"osd":2,"last update":"Tue Apr 30 09:31:13 2024","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.42199999999999999,"15min":0.40799999999999997},"min":{"1min":0.315,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.59299999999999997,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39700000000000002,"15min":0.39000000000000001},"min":{"1min":0.27200000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.54300000000000004,"5min":0.622,"15min":0.622},"last":0.36299999999999999}]},{"osd":3,"last update":"Tue Apr 30 09:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48799999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.505},{"interface":"front","average":{"1min":0.434,"5min":0.46600000000000003,"15min":0.47199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.58599999999999997,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.55800000000000005}]},{"osd":4,"last update":"Tue Apr 30 09:31:00 2024","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.64300000000000002,"15min":0.64500000000000002},"min":{"1min":0.45900000000000002,"5min":0.439,"15min":0.439},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.64100000000000001,"15min":0.64600000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.83099999999999996,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.47999999999999998}]},{"osd":5,"last update":"Tue Apr 30 09:31:27 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.65100000000000002,"15min":0.65300000000000002},"min":{"1min":0.42799999999999999,"5min":0.375,"15min":0.375},"max":{"1min":0.872,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.626},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.65200000000000002,"15min":0.65100000000000002},"min":{"1min":0.48599999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.433}]},{"osd":6,"last update":"Tue Apr 30 09:30:44 2024","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.94699999999999995,"5min":0.94699999999999995,"15min":0.94699999999999995},"last":0.51800000000000002}]},{"osd":7,"last update":"Tue Apr 30 09:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.64100000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6916,"kb_used_data":1084,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738236,"statfs":{"total":95995035648,"available":95987953664,"internally_reserved":0,"allocated":1110016,"data_stored":766504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Tue Apr 30 09:30:51 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.36499999999999999,"15min":0.34699999999999998},"min":{"1min":0.23200000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.38900000000000001,"15min":0.36299999999999999},"min":{"1min":0.28100000000000003,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.29299999999999998}]},{"osd":2,"last update":"Tue Apr 30 09:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.42099999999999999,"15min":0.40999999999999998},"min":{"1min":0.33600000000000002,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.41899999999999998,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.58399999999999996,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.435}]},{"osd":3,"last update":"Tue Apr 30 09:30:38 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.47699999999999998,"15min":0.47899999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.58199999999999996,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.47699999999999998,"15min":0.47599999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.64400000000000002,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.46600000000000003}]},{"osd":4,"last update":"Tue Apr 30 09:30:57 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.56899999999999995},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.77500000000000002,"5min":0.872,"15min":0.872},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.40799999999999997,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.99299999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"last":0.41999999999999998}]},{"osd":5,"last update":"Tue Apr 30 09:31:27 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.621,"15min":0.623},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.80600000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.501},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.59099999999999997,"15min":0.59299999999999997},"min":{"1min":0.41199999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.70799999999999996,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.41199999999999998}]},{"osd":6,"last update":"Tue Apr 30 09:30:45 2024","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.52800000000000002}]},{"osd":7,"last update":"Tue Apr 30 09:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.37}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":6272,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5696,"kb_avail":93738880,"statfs":{"total":95995035648,"available":95988613120,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5832704},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.377,"15min":0.36099999999999999},"min":{"1min":0.27500000000000002,"5min":0.193,"15min":0.193},"max":{"1min":0.79400000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.40100000000000002,"15min":0.379},"min":{"1min":0.26900000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.377}]},{"osd":1,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41099999999999998,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.82899999999999996,"5min":1.0149999999999999,"15min":1.0149999999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.438,"5min":0.41699999999999998,"15min":0.39700000000000002},"min":{"1min":0.29799999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.69499999999999995,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.68400000000000005}]},{"osd":3,"last update":"Tue Apr 30 09:31:38 2024","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48799999999999999,"15min":0.48999999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.98599999999999999,"5min":1.044,"15min":1.044},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.496,"15min":0.495},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.45500000000000002}]},{"osd":4,"last update":"Tue Apr 30 09:31:01 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.627,"15min":0.626},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.63800000000000001,"15min":0.63700000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.871,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.60599999999999998}]},{"osd":5,"last update":"Tue Apr 30 09:31:21 2024","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66600000000000004,"15min":0.66600000000000004},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.68000000000000005,"15min":0.68200000000000005},"min":{"1min":0.48899999999999999,"5min":0.44,"15min":0.44},"max":{"1min":1.1160000000000001,"5min":1.1160000000000001,"15min":1.1160000000000001},"last":0.63600000000000001}]},{"osd":6,"last update":"Tue Apr 30 09:30:45 2024","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.55600000000000005}]},{"osd":7,"last update":"Tue Apr 30 09:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.0920000000000001,"5min":1.0920000000000001,"15min":1.0920000000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.71299999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182439,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5696,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739456,"statfs":{"total":95995035648,"available":95989202944,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:31:37 2024","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":0.41299999999999998,"15min":0.41399999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.67100000000000004,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.43099999999999999,"15min":0.44500000000000001},"min":{"1min":0.28699999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.63600000000000001,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.32500000000000001}]},{"osd":1,"last update":"Tue Apr 30 09:31:37 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.44600000000000001,"15min":0.45400000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.66500000000000004,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.44800000000000001,"15min":0.45600000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.34100000000000003}]},{"osd":2,"last update":"Tue Apr 30 09:31:37 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.48299999999999998,"15min":0.49099999999999999},"min":{"1min":0.34200000000000003,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.46200000000000002,"15min":0.47999999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.59899999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.35399999999999998}]},{"osd":4,"last update":"Tue Apr 30 09:30:57 2024","interfaces":[{"interface":"back","average":{"1min":0.622,"5min":0.64100000000000001,"15min":0.64400000000000002},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.92700000000000005,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63500000000000001,"15min":0.63400000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":0.92600000000000005,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.45800000000000002}]},{"osd":5,"last update":"Tue Apr 30 09:31:22 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.66500000000000004,"15min":0.67300000000000004},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.88800000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.68200000000000005,"15min":0.69299999999999995},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.97699999999999998,"5min":1.155,"15min":1.155},"last":0.53500000000000003}]},{"osd":6,"last update":"Tue Apr 30 09:30:46 2024","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.496},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.499,"5min":0.499,"15min":0.499},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.55200000000000005}]},{"osd":7,"last update":"Tue Apr 30 09:31:10 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.436},{"interface":"front","average":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.57799999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018915,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":237841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.61399999999999999,"15min":0.63100000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.74299999999999999,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.60099999999999998,"15min":0.61599999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.65300000000000002}]},{"osd":1,"last update":"Tue Apr 30 09:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.57699999999999996,"15min":0.58199999999999996},"min":{"1min":0.41399999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.78000000000000003,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.625,"15min":0.63900000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.72099999999999997,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.52900000000000003}]},{"osd":2,"last update":"Tue Apr 30 09:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.59899999999999998,"15min":0.61099999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.61099999999999999,"15min":0.627},"min":{"1min":0.38700000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.75800000000000001,"5min":0.88,"15min":0.88},"last":0.58399999999999996}]},{"osd":3,"last update":"Tue Apr 30 09:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.59499999999999997,"15min":0.60799999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.70299999999999996,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.63800000000000001,"15min":0.65400000000000003},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69799999999999995,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.60899999999999999}]},{"osd":5,"last update":"Tue Apr 30 09:31:22 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.439,"15min":0.432},"min":{"1min":0.35699999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.504},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.42799999999999999,"15min":0.41699999999999998},"min":{"1min":0.34300000000000003,"5min":0.313,"15min":0.313},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.622}]},{"osd":6,"last update":"Tue Apr 30 09:30:42 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.70299999999999996}]},{"osd":7,"last update":"Tue Apr 30 09:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.91000000000000003,"5min":0.91000000000000003,"15min":0.91000000000000003},"last":0.52000000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":0,"num_osds":1,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":93745152,"kb_used":5684,"kb_used_data":556,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93739468,"statfs":{"total":95995035648,"available":95989215232,"internally_reserved":0,"allocated":569344,"data_stored":236988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.54200000000000004,"15min":0.54600000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.438},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.54300000000000004,"15min":0.54700000000000004},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.62,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.41299999999999998}]},{"osd":1,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.504},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.57199999999999995,"15min":0.57999999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.73799999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.38600000000000001}]},{"osd":2,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.75800000000000001,"5min":0.999,"15min":0.999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.56200000000000006,"15min":0.56699999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.67100000000000004,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.45200000000000001}]},{"osd":3,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.56799999999999995,"15min":0.57199999999999995},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.56499999999999995,"15min":0.56899999999999995},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.72299999999999998,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.48299999999999998}]},{"osd":4,"last update":"Tue Apr 30 09:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.42099999999999999,"15min":0.41199999999999998},"min":{"1min":0.28899999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.46000000000000002,"15min":0.45800000000000002},"min":{"1min":0.30299999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.499}]},{"osd":6,"last update":"Tue Apr 30 09:30:45 2024","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.54400000000000004}]},{"osd":7,"last update":"Tue Apr 30 09:31:14 2024","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.83799999999999997,"5min":0.83799999999999997,"15min":0.83799999999999997},"last":0.52300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":532480,"data_stored":528928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-30T09:31:42.255 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-30T09:31:42.255 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-30T09:31:42.255 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-30T09:31:42.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- ceph health --format=json 2024-04-30T09:31:42.484 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/mon.a/config 2024-04-30T09:31:42.575 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:31:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:42] "GET /metrics HTTP/1.1" 200 191465 "" "Prometheus/2.33.4" 2024-04-30T09:31:42.825 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:42 smithi046 ceph-mon[23181]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:42.825 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:42 smithi046 ceph-mon[23181]: from='client.14643 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-30T09:31:42.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:42 smithi026 ceph-mon[27123]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:42.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:42 smithi026 ceph-mon[27123]: from='client.14643 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-30T09:31:42.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:42 smithi026 ceph-mon[22362]: pgmap v40: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:42.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:42 smithi026 ceph-mon[22362]: from='client.14643 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-30T09:31:43.196 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:42.893Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002874796s 2024-04-30T09:31:44.222 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:31:44.223 INFO:teuthology.orchestra.run.smithi026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-30T09:31:44.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:44 smithi026 ceph-mon[27123]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:44.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:44 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1828733017' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-30T09:31:44.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:44 smithi026 ceph-mon[22362]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:44.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:44 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1828733017' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-30T09:31:44.941 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:44 smithi046 ceph-mon[23181]: pgmap v41: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:44.941 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:44 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1828733017' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-30T09:31:44.955 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-30T09:31:44.956 INFO:tasks.cephadm:Setup complete, yielding 2024-04-30T09:31:44.956 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-30T09:31:44.967 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2024-04-30T09:31:44.967 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'radosgw-admin realm create --rgw-realm=r --default' 2024-04-30T09:31:45.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:31:45.926 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_login_attempt_username - v2" 2024-04-30T09:31:46.234 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="copy login_attempt v1 to v2" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop login_attempt_tmp_qwerty" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth table" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="alter user_auth.auth_id to length 190" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth access token to user_auth" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth refresh token to user_auth" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth token type to user_auth" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add OAuth expiry to user_auth" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add index to user_id column in user_auth" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create server_lock table" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index server_lock.operation_uid" 2024-04-30T09:31:46.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create user auth token table" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.auth_token" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_auth_token.prev_auth_token" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_auth_token.user_id" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add revoked_at to the user auth token" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create cache_data table" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index cache_data.cache_key" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create short_url table v1" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index short_url.org_id-uid" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition table" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition table" 2024-04-30T09:31:46.236 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and title columns" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition on org_id and uid columns" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition table data column to mediumtext in mysql" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and title columns" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop index in alert_definition on org_id and uid columns" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and title columns" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index in alert_definition on org_id and uid columns" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column paused in alert_definition" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition table" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="delete alert_definition_version table" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="recreate alert_definition_version table" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-30T09:31:46.237 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="drop alert_definition_version table" 2024-04-30T09:31:46.238 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_instance table" 2024-04-30T09:31:46.238 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column current_state_end to alert_instance" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="remove index def_org_id, current_state on alert_instance" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_org_id to rule_org_id in alert_instance" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="rename def_uid to rule_uid in alert_instance" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index rule_org_id, current_state on alert_instance" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule table" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and title columns" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id and uid columns" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule table data column to mediumtext in mysql" 2024-04-30T09:31:46.239 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="remove unique index from alert_rule on org_id, title columns" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add dashboard_uid column to alert_rule" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add panel_id column to alert_rule" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create alert_rule_version table" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column for to alert_rule_version" 2024-04-30T09:31:46.240 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column annotations to alert_rule_version" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column labels to alert_rule_version" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id=create_alert_configuration_table 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column default in alert_configuration" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column org_id in alert_configuration" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in alert_configuration table on org_id column" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id=create_ngalert_configuration_table 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index in ngalert_configuration on org_id column" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="clear migration entry \"remove unified alerting data\"" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="move dashboard alerts to unified alerting" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element table v1" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element org_id-folder_id-name-kind" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create library_element_connection table v1" 2024-04-30T09:31:46.241 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index library_element_connection element_id-kind-connection_id" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index library_element org_id_uid" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="clone move dashboard alerts to unified alerting" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create data_keys table" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create kv_store table v1" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index kv_store.org_id-namespace-key" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="update dashboard_uid and panel_id from existing annotations" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create permission table" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index permission.role_id" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_id_action_scope" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create role table" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column display_name" 2024-04-30T09:31:46.242 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add column group_name" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index role.org_id" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role_org_id_name" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index role_org_id_uid" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create team role table" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.org_id" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index team_role_org_id_team_id_role_id" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index team_role.team_id" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create user role table" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.org_id" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index user_role_org_id_user_id_role_id" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index user_role.user_id" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create builtin role table" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.role_id" 2024-04-30T09:31:46.243 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.name" 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Add column org_id to builtin_role table" 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add index builtin_role.org_id" 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_org_id_role_id_role" 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="Remove unique index role_org_id_uid" 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index role.uid" 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="create seed assignment table" 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Executing migration" logger=migrator id="add unique index builtin_role_role_name" 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="migrations completed" logger=migrator performed=381 skipped=0 duration=190.277038ms 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:45+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:46+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:46+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:46+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:46+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-30T09:31:46.244 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:46+0000 lvl=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" logger=provisioning.datasources filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-04-30T09:31:46.245 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:46+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-30T09:31:46.245 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:46+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-30T09:31:46.245 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:46+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-30T09:31:46.245 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:31:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:31:46+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-30T09:31:47.326 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[27123]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:47.326 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:47.326 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:31:47.326 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:31:47.326 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:31:47.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[22362]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:47.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:47.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:31:47.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:31:47.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:47 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:31:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:47 smithi046 ceph-mon[23181]: pgmap v42: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:47 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:47 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:31:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:47 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:31:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:47 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:31:48.240 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:31:48.240 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "5a286eef-7c35-49b1-baa0-f540ab93020d", 2024-04-30T09:31:48.240 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "r", 2024-04-30T09:31:48.240 INFO:teuthology.orchestra.run.smithi026.stdout: "current_period": "7e71f8d0-b5d2-4550-b109-e0fc3fa46d59", 2024-04-30T09:31:48.240 INFO:teuthology.orchestra.run.smithi026.stdout: "epoch": 1 2024-04-30T09:31:48.240 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:31:48.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:48 smithi046 ceph-mon[23181]: osdmap e60: 8 total, 8 up, 8 in 2024-04-30T09:31:48.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:48 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1648610056' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-30T09:31:48.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:48 smithi046 ceph-mon[23181]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:48.330 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:48 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:48 smithi026 ceph-mon[22362]: osdmap e60: 8 total, 8 up, 8 in 2024-04-30T09:31:48.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:48 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1648610056' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-30T09:31:48.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:48 smithi026 ceph-mon[22362]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:48.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:48 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:48.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:48 smithi026 ceph-mon[27123]: osdmap e60: 8 total, 8 up, 8 in 2024-04-30T09:31:48.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:48 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1648610056' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]: dispatch 2024-04-30T09:31:48.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:48 smithi026 ceph-mon[27123]: pgmap v44: 33 pgs: 32 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:31:48.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:48 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:49.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'radosgw-admin zonegroup create --rgw-zonegroup=default --master --default' 2024-04-30T09:31:49.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:49 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1648610056' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-30T09:31:49.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:49 smithi026 ceph-mon[22362]: osdmap e61: 8 total, 8 up, 8 in 2024-04-30T09:31:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:49 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1648610056' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-30T09:31:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:49 smithi026 ceph-mon[27123]: osdmap e61: 8 total, 8 up, 8 in 2024-04-30T09:31:49.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:49 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1648610056' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": ".rgw.root","app": "rgw"}]': finished 2024-04-30T09:31:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:49 smithi046 ceph-mon[23181]: osdmap e61: 8 total, 8 up, 8 in 2024-04-30T09:31:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:50 smithi026 ceph-mon[27123]: osdmap e62: 8 total, 8 up, 8 in 2024-04-30T09:31:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:50 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:50.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:50 smithi026 ceph-mon[27123]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-30T09:31:50.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:50 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:50 smithi026 ceph-mon[22362]: osdmap e62: 8 total, 8 up, 8 in 2024-04-30T09:31:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:50 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:50 smithi026 ceph-mon[22362]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-30T09:31:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:50 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:50.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:50 smithi046 ceph-mon[23181]: osdmap e62: 8 total, 8 up, 8 in 2024-04-30T09:31:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:50 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:50 smithi046 ceph-mon[23181]: pgmap v47: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-30T09:31:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:50 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:51.306 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:51 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:51.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:51 smithi026 ceph-mon[27123]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-30T09:31:51.307 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:51 smithi026 ceph-mon[27123]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-30T09:31:51.307 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:51 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:51.307 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:51 smithi026 ceph-mon[22362]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-30T09:31:51.307 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:51 smithi026 ceph-mon[22362]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-30T09:31:51.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:51 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:31:51.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:51 smithi046 ceph-mon[23181]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-30T09:31:51.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:51 smithi046 ceph-mon[23181]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-30T09:31:51.810 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:31:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "e207fddf-2a34-4b0a-9983-8e8e79587466", 2024-04-30T09:31:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "default", 2024-04-30T09:31:51.810 INFO:teuthology.orchestra.run.smithi026.stdout: "api_name": "default", 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "is_master": "true", 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "endpoints": [], 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "hostnames": [], 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "hostnames_s3website": [], 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "master_zone": "", 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "zones": [], 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "placement_targets": [], 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "default_placement": "", 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_id": "5a286eef-7c35-49b1-baa0-f540ab93020d", 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_policy": { 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: "groups": [] 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:31:51.811 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:31:52.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:52 smithi026 ceph-mon[27123]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-30T09:31:52.389 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:31:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:52] "GET /metrics HTTP/1.1" 200 191469 "" "Prometheus/2.33.4" 2024-04-30T09:31:52.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:52 smithi026 ceph-mon[22362]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-30T09:31:52.413 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default' 2024-04-30T09:31:52.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:52 smithi046 ceph-mon[23181]: pgmap v48: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-30T09:31:54.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:54 smithi026 ceph-mon[27123]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-30T09:31:54.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:54 smithi026 ceph-mon[22362]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-30T09:31:54.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:54 smithi046 ceph-mon[23181]: pgmap v49: 33 pgs: 33 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 1 op/s 2024-04-30T09:31:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:31:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:31:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:31:56.054 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:55 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:31:56.055 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:55 smithi026 bash[46784]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-30T09:31:56.375 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[45148]: level=info ts=2024-04-30T09:31:56.250Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-30T09:31:56.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:56 smithi026 ceph-mon[27123]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-30T09:31:56.695 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:56 smithi026 podman[46794]: 2024-04-30 09:31:56.375079776 +0000 UTC m=+0.425318395 container died 354b3c369536731fd7173078d750c61a87d50cb80b8a837fde391c059e2bb2d4 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:31:56.695 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:56 smithi026 ceph-mon[22362]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-30T09:31:56.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:56 smithi026 podman[46794]: 2024-04-30 09:31:56.888116309 +0000 UTC m=+0.938354930 container cleanup 354b3c369536731fd7173078d750c61a87d50cb80b8a837fde391c059e2bb2d4 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:31:56.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:56 smithi026 bash[46794]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a 2024-04-30T09:31:56.980 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:31:56.980 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "c7c6ee7b-b3b7-4932-9f46-deddd8552c91", 2024-04-30T09:31:56.980 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "z", 2024-04-30T09:31:56.980 INFO:teuthology.orchestra.run.smithi026.stdout: "domain_root": "z.rgw.meta:root", 2024-04-30T09:31:56.980 INFO:teuthology.orchestra.run.smithi026.stdout: "control_pool": "z.rgw.control", 2024-04-30T09:31:56.980 INFO:teuthology.orchestra.run.smithi026.stdout: "gc_pool": "z.rgw.log:gc", 2024-04-30T09:31:56.980 INFO:teuthology.orchestra.run.smithi026.stdout: "lc_pool": "z.rgw.log:lc", 2024-04-30T09:31:56.980 INFO:teuthology.orchestra.run.smithi026.stdout: "log_pool": "z.rgw.log", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "intent_log_pool": "z.rgw.log:intent", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "usage_log_pool": "z.rgw.log:usage", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "roles_pool": "z.rgw.meta:roles", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "reshard_pool": "z.rgw.log:reshard", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "user_keys_pool": "z.rgw.meta:users.keys", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "user_email_pool": "z.rgw.meta:users.email", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "user_swift_pool": "z.rgw.meta:users.swift", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "user_uid_pool": "z.rgw.meta:users.uid", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "otp_pool": "z.rgw.otp", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "system_key": { 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "access_key": "", 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "secret_key": "" 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:31:56.981 INFO:teuthology.orchestra.run.smithi026.stdout: "placement_pools": [ 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: "key": "default-placement", 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: "val": { 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: "index_pool": "z.rgw.buckets.index", 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: "storage_classes": { 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: "STANDARD": { 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: "data_pool": "z.rgw.buckets.data" 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: "data_extra_pool": "z.rgw.buckets.non-ec", 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: "index_type": 0 2024-04-30T09:31:56.982 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:31:56.983 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:31:56.983 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:31:56.983 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_id": "5a286eef-7c35-49b1-baa0-f540ab93020d", 2024-04-30T09:31:56.983 INFO:teuthology.orchestra.run.smithi026.stdout: "notif_pool": "z.rgw.log:notif" 2024-04-30T09:31:56.983 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:31:56.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:56 smithi046 ceph-mon[23181]: pgmap v50: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 1.4 KiB/s wr, 2 op/s 2024-04-30T09:31:57.723 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'radosgw-admin period update --rgw-realm=r --commit' 2024-04-30T09:31:57.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:57 smithi026 podman[46814]: 2024-04-30 09:31:57.918778804 +0000 UTC m=+1.641483110 container remove 354b3c369536731fd7173078d750c61a87d50cb80b8a837fde391c059e2bb2d4 (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:31:58.263 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:58 smithi026 podman[46814]: 2024-04-30 09:31:58.002744558 +0000 UTC m=+1.725448865 volume remove 5f5134ec4070d815ecd8de8e0ccf1be866c1566d5940c74554df06a50274dc21 2024-04-30T09:31:58.263 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:58 smithi026 bash[46903]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-30T09:31:58.263 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:58 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-30T09:31:58.263 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:58 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:31:58.264 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:58 smithi026 systemd[1]: Starting Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:31:58.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:31:58 smithi026 ceph-mon[22362]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 439 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-04-30T09:31:58.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:31:58 smithi026 ceph-mon[27123]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 439 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-04-30T09:31:58.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:31:58 smithi046 ceph-mon[23181]: pgmap v51: 33 pgs: 33 active+clean; 578 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 439 B/s rd, 1.2 KiB/s wr, 1 op/s 2024-04-30T09:31:59.342 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 podman[47042]: 2024-04-30 09:31:59.091734883 +0000 UTC m=+0.021348824 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-30T09:31:59.342 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 podman[47042]: 2024-04-30 09:31:59.22498884 +0000 UTC m=+0.154602765 volume create c48ee1fc834c96cd1b6660f45e55129d2c5a6376b48b859a5939df5e0db3d8e2 2024-04-30T09:31:59.342 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 podman[47042]: 2024-04-30 09:31:59.342221962 +0000 UTC m=+0.271835928 container create 7b0e3dcf393a466e6330dab6626416cb1ae0851c5a31b0c925efb048397db8cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:31:59.898 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 podman[47042]: 2024-04-30 09:31:59.645604362 +0000 UTC m=+0.575218286 container init 7b0e3dcf393a466e6330dab6626416cb1ae0851c5a31b0c925efb048397db8cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:31:59.898 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 podman[47042]: 2024-04-30 09:31:59.6487887 +0000 UTC m=+0.578402627 container start 7b0e3dcf393a466e6330dab6626416cb1ae0851c5a31b0c925efb048397db8cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:31:59.898 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:31:59.660Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-30T09:31:59.899 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:31:59.660Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-30T09:31:59.899 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:31:59.662Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-04-30T09:31:59.899 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:31:59.663Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-30T09:31:59.899 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:31:59.689Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-30T09:31:59.899 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:31:59.689Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-30T09:31:59.899 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:31:59.692Z caller=main.go:518 msg=Listening address=:9093 2024-04-30T09:31:59.899 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:31:59.692Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-30T09:32:00.196 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 bash[47042]: 7b0e3dcf393a466e6330dab6626416cb1ae0851c5a31b0c925efb048397db8cb 2024-04-30T09:32:00.197 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:31:59 smithi026 systemd[1]: Started Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:32:01.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:00 smithi026 ceph-mon[22362]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 793 B/s wr, 1 op/s 2024-04-30T09:32:01.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:00 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:01.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:00 smithi026 ceph-mon[22362]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:32:01.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:00 smithi026 ceph-mon[22362]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:32:01.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:00 smithi026 ceph-mon[27123]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 793 B/s wr, 1 op/s 2024-04-30T09:32:01.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:00 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:01.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:00 smithi026 ceph-mon[27123]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:32:01.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:00 smithi026 ceph-mon[27123]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:32:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:00 smithi046 ceph-mon[23181]: pgmap v52: 33 pgs: 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 694 B/s rd, 793 B/s wr, 1 op/s 2024-04-30T09:32:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:00 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:00 smithi046 ceph-mon[23181]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:32:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:00 smithi046 ceph-mon[23181]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:32:01.939 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:32:01 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:32:01.663Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000311615s 2024-04-30T09:32:02.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:01 smithi026 ceph-mon[22362]: osdmap e63: 8 total, 8 up, 8 in 2024-04-30T09:32:02.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:01 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-30T09:32:02.197 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:01 smithi046 ceph-mon[23181]: osdmap e63: 8 total, 8 up, 8 in 2024-04-30T09:32:02.197 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:01 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-30T09:32:02.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:01 smithi026 ceph-mon[27123]: osdmap e63: 8 total, 8 up, 8 in 2024-04-30T09:32:02.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:01 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]: dispatch 2024-04-30T09:32:02.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:32:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:02] "GET /metrics HTTP/1.1" 200 192969 "" "Prometheus/2.33.4" 2024-04-30T09:32:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:02 smithi046 ceph-mon[23181]: pgmap v54: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 818 B/s wr, 1 op/s 2024-04-30T09:32:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:02 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-30T09:32:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:02 smithi046 ceph-mon[23181]: osdmap e64: 8 total, 8 up, 8 in 2024-04-30T09:32:03.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:02 smithi026 ceph-mon[22362]: pgmap v54: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 818 B/s wr, 1 op/s 2024-04-30T09:32:03.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:02 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-30T09:32:03.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:02 smithi026 ceph-mon[22362]: osdmap e64: 8 total, 8 up, 8 in 2024-04-30T09:32:03.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:02 smithi026 ceph-mon[27123]: pgmap v54: 65 pgs: 32 unknown, 33 active+clean; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 716 B/s rd, 818 B/s wr, 1 op/s 2024-04-30T09:32:03.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:02 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.log","app": "rgw"}]': finished 2024-04-30T09:32:03.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:02 smithi026 ceph-mon[27123]: osdmap e64: 8 total, 8 up, 8 in 2024-04-30T09:32:03.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:03 smithi026 ceph-mon[27123]: osdmap e65: 8 total, 8 up, 8 in 2024-04-30T09:32:03.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:03 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-30T09:32:03.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:03 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-30T09:32:03.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:03 smithi026 ceph-mon[22362]: osdmap e65: 8 total, 8 up, 8 in 2024-04-30T09:32:03.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:03 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-30T09:32:03.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:03 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-30T09:32:03.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:03 smithi026 ceph-mon[22362]: osdmap e66: 8 total, 8 up, 8 in 2024-04-30T09:32:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:03 smithi046 ceph-mon[23181]: osdmap e65: 8 total, 8 up, 8 in 2024-04-30T09:32:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:03 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]: dispatch 2024-04-30T09:32:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:03 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4060896935' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.control","app": "rgw"}]': finished 2024-04-30T09:32:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:03 smithi046 ceph-mon[23181]: osdmap e66: 8 total, 8 up, 8 in 2024-04-30T09:32:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:03 smithi026 ceph-mon[27123]: osdmap e66: 8 total, 8 up, 8 in 2024-04-30T09:32:05.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:04 smithi046 ceph-mon[23181]: pgmap v57: 97 pgs: 6 creating+peering, 39 active+clean, 52 unknown; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 853 B/s wr, 2 op/s 2024-04-30T09:32:05.137 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:32:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:32:05.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:04 smithi026 ceph-mon[27123]: pgmap v57: 97 pgs: 6 creating+peering, 39 active+clean, 52 unknown; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 853 B/s wr, 2 op/s 2024-04-30T09:32:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:04 smithi026 ceph-mon[22362]: pgmap v57: 97 pgs: 6 creating+peering, 39 active+clean, 52 unknown; 579 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 853 B/s wr, 2 op/s 2024-04-30T09:32:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:05 smithi046 ceph-mon[23181]: osdmap e67: 8 total, 8 up, 8 in 2024-04-30T09:32:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:05 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-30T09:32:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:05 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-30T09:32:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:05 smithi046 ceph-mon[23181]: osdmap e68: 8 total, 8 up, 8 in 2024-04-30T09:32:06.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:05 smithi026 ceph-mon[22362]: osdmap e67: 8 total, 8 up, 8 in 2024-04-30T09:32:06.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:05 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-30T09:32:06.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:05 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-30T09:32:06.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:05 smithi026 ceph-mon[22362]: osdmap e68: 8 total, 8 up, 8 in 2024-04-30T09:32:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:05 smithi026 ceph-mon[27123]: osdmap e67: 8 total, 8 up, 8 in 2024-04-30T09:32:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:05 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]: dispatch 2024-04-30T09:32:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:05 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "z.rgw.meta","app": "rgw"}]': finished 2024-04-30T09:32:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:05 smithi026 ceph-mon[27123]: osdmap e68: 8 total, 8 up, 8 in 2024-04-30T09:32:06.539 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 systemd[1]: Stopping Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:32:06.539 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 bash[39991]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:32:06.823 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.538Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-30T09:32:06.823 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.538Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-30T09:32:06.823 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.538Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-30T09:32:06.823 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.538Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-30T09:32:06.823 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.538Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-30T09:32:06.823 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.538Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-30T09:32:06.823 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.538Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-30T09:32:06.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.538Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-30T09:32:06.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.538Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-30T09:32:06.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.539Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-30T09:32:06.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.539Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-30T09:32:06.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[38302]: ts=2024-04-30T09:32:06.539Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-30T09:32:06.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 podman[40003]: 2024-04-30 09:32:06.548684295 +0000 UTC m=+0.102190224 container died 128e33ff71d06e63c3c359124b3593b5d81f76aaf84c1424eedefd202811d7c4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:32:07.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 podman[40003]: 2024-04-30 09:32:06.921015649 +0000 UTC m=+0.474521582 container cleanup 128e33ff71d06e63c3c359124b3593b5d81f76aaf84c1424eedefd202811d7c4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:32:07.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 bash[40003]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a 2024-04-30T09:32:07.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:06 smithi046 bash[40031]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:32:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-mon[23181]: pgmap v60: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-30T09:32:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-30T09:32:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-30T09:32:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-mon[23181]: osdmap e69: 8 total, 8 up, 8 in 2024-04-30T09:32:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:06 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-30T09:32:07.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[22362]: pgmap v60: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-30T09:32:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-30T09:32:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-30T09:32:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[22362]: osdmap e69: 8 total, 8 up, 8 in 2024-04-30T09:32:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-30T09:32:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[27123]: pgmap v60: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.7 KiB/s rd, 5 op/s 2024-04-30T09:32:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]: dispatch 2024-04-30T09:32:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_autoscale_bias", "val": "4"}]': finished 2024-04-30T09:32:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[27123]: osdmap e69: 8 total, 8 up, 8 in 2024-04-30T09:32:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:06 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]: dispatch 2024-04-30T09:32:07.606 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 podman[40017]: 2024-04-30 09:32:07.330808668 +0000 UTC m=+0.784530715 container remove 128e33ff71d06e63c3c359124b3593b5d81f76aaf84c1424eedefd202811d7c4 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:32:07.607 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-30T09:32:07.607 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 systemd[1]: Stopped Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:32:07.607 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 systemd[1]: Starting Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:32:07.607 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 podman[40101]: 2024-04-30 09:32:07.483502489 +0000 UTC m=+0.020712636 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-30T09:32:07.607 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 podman[40101]: 2024-04-30 09:32:07.606708639 +0000 UTC m=+0.143918784 container create f54f9679bceb4d6ac5a87db25cd211c06787f9a1161db5300859e5a28ec29d71 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:32:07.876 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 podman[40101]: 2024-04-30 09:32:07.800943276 +0000 UTC m=+0.338153423 container init f54f9679bceb4d6ac5a87db25cd211c06787f9a1161db5300859e5a28ec29d71 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:32:07.876 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 podman[40101]: 2024-04-30 09:32:07.803738966 +0000 UTC m=+0.340949115 container start f54f9679bceb4d6ac5a87db25cd211c06787f9a1161db5300859e5a28ec29d71 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:32:07.876 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.828Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-30T09:32:07.876 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.828Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-30T09:32:07.876 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.828Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-30T09:32:07.876 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.828Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi046 (none))" 2024-04-30T09:32:07.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.828Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-30T09:32:07.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.828Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-30T09:32:07.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.830Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-30T09:32:07.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.830Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-30T09:32:07.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.831Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-30T09:32:07.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.832Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-30T09:32:07.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.832Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.437µs 2024-04-30T09:32:07.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:07.832Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-30T09:32:07.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 bash[40101]: f54f9679bceb4d6ac5a87db25cd211c06787f9a1161db5300859e5a28ec29d71 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "5c2f19e3-ea18-4408-a6fd-38c3114412bd", 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: "epoch": 1, 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: "predecessor_uuid": "7e71f8d0-b5d2-4550-b109-e0fc3fa46d59", 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_status": [], 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: "period_map": { 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "5c2f19e3-ea18-4408-a6fd-38c3114412bd", 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: "zonegroups": [ 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "e207fddf-2a34-4b0a-9983-8e8e79587466", 2024-04-30T09:32:08.017 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "default", 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "api_name": "default", 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "is_master": "true", 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "endpoints": [], 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "hostnames": [], 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "hostnames_s3website": [], 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "master_zone": "c7c6ee7b-b3b7-4932-9f46-deddd8552c91", 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "zones": [ 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "id": "c7c6ee7b-b3b7-4932-9f46-deddd8552c91", 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "z", 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "endpoints": [], 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "log_meta": "false", 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "log_data": "false", 2024-04-30T09:32:08.018 INFO:teuthology.orchestra.run.smithi026.stdout: "bucket_index_max_shards": 11, 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "read_only": "false", 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "tier_type": "", 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_from_all": "true", 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_from": [], 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "redirect_zone": "" 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "placement_targets": [ 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "default-placement", 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "tags": [], 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "storage_classes": [ 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: "STANDARD" 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:32:08.019 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: "default_placement": "default-placement", 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_id": "5a286eef-7c35-49b1-baa0-f540ab93020d", 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: "sync_policy": { 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: "groups": [] 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: "short_zone_ids": [ 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: "key": "c7c6ee7b-b3b7-4932-9f46-deddd8552c91", 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: "val": 2964062038 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:32:08.020 INFO:teuthology.orchestra.run.smithi026.stdout: "master_zonegroup": "e207fddf-2a34-4b0a-9983-8e8e79587466", 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "master_zone": "c7c6ee7b-b3b7-4932-9f46-deddd8552c91", 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "period_config": { 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "bucket_quota": { 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false, 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "check_on_raw": false, 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "max_size": -1, 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "max_size_kb": 0, 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "max_objects": -1 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "user_quota": { 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false, 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "check_on_raw": false, 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "max_size": -1, 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "max_size_kb": 0, 2024-04-30T09:32:08.021 INFO:teuthology.orchestra.run.smithi026.stdout: "max_objects": -1 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: "user_ratelimit": { 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_ops": 0, 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_ops": 0, 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_bytes": 0, 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_bytes": 0, 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: "bucket_ratelimit": { 2024-04-30T09:32:08.022 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_ops": 0, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_ops": 0, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_bytes": 0, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_bytes": 0, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "anonymous_ratelimit": { 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_ops": 0, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_ops": 0, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "max_read_bytes": 0, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "max_write_bytes": 0, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "enabled": false 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_id": "5a286eef-7c35-49b1-baa0-f540ab93020d", 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_name": "r", 2024-04-30T09:32:08.023 INFO:teuthology.orchestra.run.smithi026.stdout: "realm_epoch": 2 2024-04-30T09:32:08.024 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:32:08.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:07 smithi046 systemd[1]: Started Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:32:08.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:08.054Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-30T09:32:08.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:08.055Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-30T09:32:08.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:08.055Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.189µs wal_replay_duration=222.307079ms total_replay_duration=222.346611ms 2024-04-30T09:32:08.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:08.057Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-30T09:32:08.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:08.057Z caller=main.go:947 level=info msg="TSDB started" 2024-04-30T09:32:08.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:08.057Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-30T09:32:08.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:08.075Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.927813ms db_storage=711ns remote_storage=1.289µs web_handler=309ns query_engine=667ns scrape=347.338µs scrape_sd=22.75µs notify=20.386µs notify_sd=7.788µs rules=17.262418ms 2024-04-30T09:32:08.235 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:32:08.075Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-30T09:32:08.813 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000' 2024-04-30T09:32:09.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: pgmap v63: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 4 op/s 2024-04-30T09:32:09.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:32:09.064 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.46:9095"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.46:9095"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.46:3000"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.46:3000"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:09.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:09.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:09.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-30T09:32:09.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[27123]: osdmap e70: 8 total, 8 up, 8 in 2024-04-30T09:32:09.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: pgmap v63: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 4 op/s 2024-04-30T09:32:09.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.46:9095"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.46:9095"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.46:3000"}]: dispatch 2024-04-30T09:32:09.067 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.46:3000"}]: dispatch 2024-04-30T09:32:09.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:09.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:09.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:09.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-30T09:32:09.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:08 smithi026 ceph-mon[22362]: osdmap e70: 8 total, 8 up, 8 in 2024-04-30T09:32:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: pgmap v63: 129 pgs: 32 unknown, 32 creating+peering, 65 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 4 op/s 2024-04-30T09:32:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:32:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:32:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-30T09:32:09.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://172.21.15.26:9093"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.46:9095"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://172.21.15.46:9095"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.46:3000"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://172.21.15.46:3000"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2195427871' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "z.rgw.meta", "var": "pg_num_min", "val": "8"}]': finished 2024-04-30T09:32:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:08 smithi046 ceph-mon[23181]: osdmap e70: 8 total, 8 up, 8 in 2024-04-30T09:32:09.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:32:09 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:32:09.664Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.001595322s 2024-04-30T09:32:10.972 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:10 smithi026 ceph-mon[27123]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-04-30T09:32:10.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:10 smithi026 ceph-mon[22362]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-04-30T09:32:11.019 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:10 smithi046 ceph-mon[23181]: pgmap v65: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 2.0 KiB/s wr, 5 op/s 2024-04-30T09:32:11.448 INFO:teuthology.orchestra.run.smithi026.stdout:Scheduled rgw.foo update... 2024-04-30T09:32:12.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:12.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[22362]: from='client.14715 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:32:12.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[22362]: Saving service rgw.foo spec with placement count:2 2024-04-30T09:32:12.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:12.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[22362]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-30T09:32:12.341 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:12.341 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[27123]: from='client.14715 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:32:12.341 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[27123]: Saving service rgw.foo spec with placement count:2 2024-04-30T09:32:12.341 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:12.342 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:12 smithi026 ceph-mon[27123]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-30T09:32:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:12 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:12 smithi046 ceph-mon[23181]: from='client.14715 -' entity='client.admin' cmd=[{"prefix": "orch apply rgw", "svc_id": "foo", "realm": "r", "zone": "z", "placement": "2", "port": 8000, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:32:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:12 smithi046 ceph-mon[23181]: Saving service rgw.foo spec with placement count:2 2024-04-30T09:32:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:12 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:12 smithi046 ceph-mon[23181]: pgmap v66: 129 pgs: 129 active+clean; 581 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1.5 KiB/s wr, 3 op/s 2024-04-30T09:32:12.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph osd pool create foo' 2024-04-30T09:32:14.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-30T09:32:14.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: Saving service rgw.foo spec with placement count:2 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[27123]: Deploying daemon rgw.foo.smithi026.igccwv on smithi026 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-30T09:32:14.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: Saving service rgw.foo spec with placement count:2 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:14.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:14 smithi026 ceph-mon[22362]: Deploying daemon rgw.foo.smithi026.igccwv on smithi026 2024-04-30T09:32:14.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: pgmap v67: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1.2 KiB/s wr, 3 op/s 2024-04-30T09:32:14.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: Saving service rgw.foo spec with placement count:2 2024-04-30T09:32:14.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:14.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:14.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-30T09:32:14.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:14.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:14.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:14 smithi046 ceph-mon[23181]: Deploying daemon rgw.foo.smithi026.igccwv on smithi026 2024-04-30T09:32:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:32:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:32:15.650 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:15 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2817685671' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-30T09:32:15.651 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:15 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2817685671' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-30T09:32:15.651 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:15 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-30T09:32:15.659 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'foo' created 2024-04-30T09:32:15.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:15 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-30T09:32:15.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:15 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2817685671' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-30T09:32:15.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:15 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "foo"}]: dispatch 2024-04-30T09:32:16.323 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'rbd pool init foo' 2024-04-30T09:32:16.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:16 smithi026 ceph-mon[22362]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 1.0 KiB/s wr, 2 op/s 2024-04-30T09:32:16.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:16 smithi026 ceph-mon[27123]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 1.0 KiB/s wr, 2 op/s 2024-04-30T09:32:16.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:16 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-30T09:32:16.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:16 smithi026 ceph-mon[27123]: osdmap e71: 8 total, 8 up, 8 in 2024-04-30T09:32:16.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:16 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-30T09:32:16.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:16 smithi026 ceph-mon[22362]: osdmap e71: 8 total, 8 up, 8 in 2024-04-30T09:32:16.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:16 smithi046 ceph-mon[23181]: pgmap v68: 129 pgs: 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 1.0 KiB/s wr, 2 op/s 2024-04-30T09:32:16.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:16 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "foo"}]': finished 2024-04-30T09:32:16.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:16 smithi046 ceph-mon[23181]: osdmap e71: 8 total, 8 up, 8 in 2024-04-30T09:32:17.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:17 smithi026 ceph-mon[22362]: osdmap e72: 8 total, 8 up, 8 in 2024-04-30T09:32:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:17 smithi026 ceph-mon[27123]: osdmap e72: 8 total, 8 up, 8 in 2024-04-30T09:32:17.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:17 smithi046 ceph-mon[23181]: osdmap e72: 8 total, 8 up, 8 in 2024-04-30T09:32:18.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:18 smithi026 ceph-mon[22362]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:18.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:18 smithi026 ceph-mon[22362]: osdmap e73: 8 total, 8 up, 8 in 2024-04-30T09:32:18.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:18 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3102505222' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-30T09:32:18.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:18 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-30T09:32:18.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:18 smithi026 ceph-mon[27123]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:18.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:18 smithi026 ceph-mon[27123]: osdmap e73: 8 total, 8 up, 8 in 2024-04-30T09:32:18.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:18 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3102505222' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-30T09:32:18.928 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:18 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-30T09:32:18.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:18 smithi046 ceph-mon[23181]: pgmap v71: 161 pgs: 32 unknown, 129 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:18.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:18 smithi046 ceph-mon[23181]: osdmap e73: 8 total, 8 up, 8 in 2024-04-30T09:32:18.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:18 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3102505222' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-30T09:32:18.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:18 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]: dispatch 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[22362]: osdmap e74: 8 total, 8 up, 8 in 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[22362]: Deploying daemon rgw.foo.smithi046.wmzons on smithi046 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-30T09:32:19.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[27123]: osdmap e74: 8 total, 8 up, 8 in 2024-04-30T09:32:19.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:19.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:19.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:19.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-30T09:32:19.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:19.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:19.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:19 smithi026 ceph-mon[27123]: Deploying daemon rgw.foo.smithi046.wmzons on smithi046 2024-04-30T09:32:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:19 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "foo","app": "rbd"}]': finished 2024-04-30T09:32:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:19 smithi046 ceph-mon[23181]: osdmap e74: 8 total, 8 up, 8 in 2024-04-30T09:32:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:19 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:19 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:19 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:32:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:19 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]': finished 2024-04-30T09:32:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:19 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:19.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:19 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:19.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:19 smithi046 ceph-mon[23181]: Deploying daemon rgw.foo.smithi046.wmzons on smithi046 2024-04-30T09:32:20.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:20 smithi026 ceph-mon[22362]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:20.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:20 smithi026 ceph-mon[22362]: osdmap e75: 8 total, 8 up, 8 in 2024-04-30T09:32:20.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:20 smithi026 ceph-mon[27123]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:20.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:20 smithi026 ceph-mon[27123]: osdmap e75: 8 total, 8 up, 8 in 2024-04-30T09:32:20.966 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:20 smithi046 ceph-mon[23181]: pgmap v74: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:20.966 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:20 smithi046 ceph-mon[23181]: osdmap e75: 8 total, 8 up, 8 in 2024-04-30T09:32:21.309 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch apply iscsi foo u p' 2024-04-30T09:32:21.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[27123]: osdmap e76: 8 total, 8 up, 8 in 2024-04-30T09:32:21.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:21.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:21.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:21.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:21.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[22362]: osdmap e76: 8 total, 8 up, 8 in 2024-04-30T09:32:21.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:21.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:21.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:21.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:21 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:21.979 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:21 smithi046 ceph-mon[23181]: osdmap e76: 8 total, 8 up, 8 in 2024-04-30T09:32:21.979 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:21 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:21.979 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:21 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:21.979 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:21 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:21.979 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:21 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:22.617 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:32:22 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:22] "GET /metrics HTTP/1.1" 200 196751 "" "Prometheus/2.33.4" 2024-04-30T09:32:22.868 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:22 smithi026 ceph-mon[27123]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:22.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:22 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:22.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:22 smithi026 ceph-mon[22362]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:22.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:22 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:22.957 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:22 smithi046 ceph-mon[23181]: pgmap v77: 161 pgs: 161 active+clean; 581 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:22.957 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:22 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:23.274 INFO:teuthology.orchestra.run.smithi026.stdout:Scheduled iscsi.foo update... 2024-04-30T09:32:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:24 smithi026 ceph-mon[27123]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:32:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:24 smithi026 ceph-mon[27123]: Saving service iscsi.foo spec with placement count:1 2024-04-30T09:32:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:24 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:24 smithi026 ceph-mon[27123]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 59 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 5.3 KiB/s wr, 286 op/s 2024-04-30T09:32:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:24 smithi026 ceph-mon[22362]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:32:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:24 smithi026 ceph-mon[22362]: Saving service iscsi.foo spec with placement count:1 2024-04-30T09:32:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:24 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:24 smithi026 ceph-mon[22362]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 59 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 5.3 KiB/s wr, 286 op/s 2024-04-30T09:32:24.590 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:24 smithi046 ceph-mon[23181]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "orch apply iscsi", "pool": "foo", "api_user": "u", "api_password": "p", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:32:24.590 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:24 smithi046 ceph-mon[23181]: Saving service iscsi.foo spec with placement count:1 2024-04-30T09:32:24.590 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:24 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:24.590 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:24 smithi046 ceph-mon[23181]: pgmap v78: 161 pgs: 161 active+clean; 584 KiB data, 59 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 5.3 KiB/s wr, 286 op/s 2024-04-30T09:32:24.593 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'sleep 180' 2024-04-30T09:32:25.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:32:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:32:26.124 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:25 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:26.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:25 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:25 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:26 smithi026 ceph-mon[27123]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 162 KiB/s rd, 5.2 KiB/s wr, 309 op/s 2024-04-30T09:32:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:26 smithi026 ceph-mon[22362]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 162 KiB/s rd, 5.2 KiB/s wr, 309 op/s 2024-04-30T09:32:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:26 smithi046 ceph-mon[23181]: pgmap v79: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 162 KiB/s rd, 5.2 KiB/s wr, 309 op/s 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:32:28.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:32:28.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:32:28.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:32:28.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-30T09:32:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-30T09:32:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2024-04-30T09:32:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2024-04-30T09:32:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:32:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:32:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:32:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:32:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-30T09:32:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]: dispatch 2024-04-30T09:32:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2024-04-30T09:32:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2024-04-30T09:32:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-04-30T09:32:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:32:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:32:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:32:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:32:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:29.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[27123]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 4.5 KiB/s wr, 265 op/s 2024-04-30T09:32:29.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[27123]: Checking dashboard <-> RGW credentials 2024-04-30T09:32:29.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-04-30T09:32:29.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-30T09:32:29.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2024-04-30T09:32:29.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-30T09:32:29.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[27123]: osdmap e77: 8 total, 8 up, 8 in 2024-04-30T09:32:29.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[22362]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 4.5 KiB/s wr, 265 op/s 2024-04-30T09:32:29.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[22362]: Checking dashboard <-> RGW credentials 2024-04-30T09:32:29.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-04-30T09:32:29.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-30T09:32:29.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2024-04-30T09:32:29.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-30T09:32:29.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:28 smithi026 ceph-mon[22362]: osdmap e77: 8 total, 8 up, 8 in 2024-04-30T09:32:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:28 smithi046 ceph-mon[23181]: pgmap v80: 161 pgs: 161 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 4.5 KiB/s wr, 265 op/s 2024-04-30T09:32:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:28 smithi046 ceph-mon[23181]: Checking dashboard <-> RGW credentials 2024-04-30T09:32:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "2.1a", "id": [7, 2]}]': finished 2024-04-30T09:32:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.7", "id": [1, 2]}]': finished 2024-04-30T09:32:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2024-04-30T09:32:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-04-30T09:32:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:28 smithi046 ceph-mon[23181]: osdmap e77: 8 total, 8 up, 8 in 2024-04-30T09:32:30.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: osdmap e78: 8 total, 8 up, 8 in 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: Checking pool "foo" exists for service iscsi.foo 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[27123]: Deploying daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: osdmap e78: 8 total, 8 up, 8 in 2024-04-30T09:32:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:30.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:30.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:30.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: Checking pool "foo" exists for service iscsi.foo 2024-04-30T09:32:30.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:32:30.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:32:30.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-30T09:32:30.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:30.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:29 smithi026 ceph-mon[22362]: Deploying daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:32:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: osdmap e78: 8 total, 8 up, 8 in 2024-04-30T09:32:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: Checking pool "foo" exists for service iscsi.foo 2024-04-30T09:32:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:32:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:32:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]': finished 2024-04-30T09:32:30.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:30.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:29 smithi046 ceph-mon[23181]: Deploying daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:32:31.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:31 smithi026 ceph-mon[27123]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 142 KiB/s rd, 4.5 KiB/s wr, 268 op/s 2024-04-30T09:32:31.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:31 smithi026 ceph-mon[27123]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-30T09:32:31.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:31 smithi026 ceph-mon[27123]: osdmap e79: 8 total, 8 up, 8 in 2024-04-30T09:32:31.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:31 smithi026 ceph-mon[22362]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 142 KiB/s rd, 4.5 KiB/s wr, 268 op/s 2024-04-30T09:32:31.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:31 smithi026 ceph-mon[22362]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-30T09:32:31.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:31 smithi026 ceph-mon[22362]: osdmap e79: 8 total, 8 up, 8 in 2024-04-30T09:32:31.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:31 smithi046 ceph-mon[23181]: pgmap v83: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 142 KiB/s rd, 4.5 KiB/s wr, 268 op/s 2024-04-30T09:32:31.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:31 smithi046 ceph-mon[23181]: mgrmap e20: y(active, since 2m), standbys: x 2024-04-30T09:32:31.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:31 smithi046 ceph-mon[23181]: osdmap e79: 8 total, 8 up, 8 in 2024-04-30T09:32:32.683 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:32:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:32] "GET /metrics HTTP/1.1" 200 207872 "" "Prometheus/2.33.4" 2024-04-30T09:32:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:33 smithi046 ceph-mon[23181]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 3 op/s 2024-04-30T09:32:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:33 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:33 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:33 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:33 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:33 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:33 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2489880525' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:32:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:33 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3825969446' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/209764474"}]: dispatch 2024-04-30T09:32:33.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[27123]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 3 op/s 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2489880525' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3825969446' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/209764474"}]: dispatch 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[22362]: pgmap v85: 161 pgs: 4 peering, 157 active+clean; 584 KiB data, 60 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 3 op/s 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:33.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:33.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:33.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:33.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2489880525' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:32:33.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:33 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3825969446' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/209764474"}]: dispatch 2024-04-30T09:32:34.710 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:34 smithi046 ceph-mon[23181]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 511 B/s wr, 9 op/s; 0 B/s, 0 objects/s recovering 2024-04-30T09:32:34.710 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:34 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3825969446' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/209764474"}]': finished 2024-04-30T09:32:34.710 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:34 smithi046 ceph-mon[23181]: osdmap e80: 8 total, 8 up, 8 in 2024-04-30T09:32:34.710 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:34 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2300038731' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/753133130"}]: dispatch 2024-04-30T09:32:34.710 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:34 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/753133130"}]: dispatch 2024-04-30T09:32:34.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[27123]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 511 B/s wr, 9 op/s; 0 B/s, 0 objects/s recovering 2024-04-30T09:32:34.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3825969446' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/209764474"}]': finished 2024-04-30T09:32:34.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[27123]: osdmap e80: 8 total, 8 up, 8 in 2024-04-30T09:32:34.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2300038731' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/753133130"}]: dispatch 2024-04-30T09:32:34.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/753133130"}]: dispatch 2024-04-30T09:32:34.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[22362]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 7.3 KiB/s rd, 511 B/s wr, 9 op/s; 0 B/s, 0 objects/s recovering 2024-04-30T09:32:34.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3825969446' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/209764474"}]': finished 2024-04-30T09:32:34.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[22362]: osdmap e80: 8 total, 8 up, 8 in 2024-04-30T09:32:34.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2300038731' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/753133130"}]: dispatch 2024-04-30T09:32:34.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:34 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/753133130"}]: dispatch 2024-04-30T09:32:35.372 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:32:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:32:35.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:35 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/753133130"}]': finished 2024-04-30T09:32:35.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:35 smithi046 ceph-mon[23181]: osdmap e81: 8 total, 8 up, 8 in 2024-04-30T09:32:35.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:35 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/844025147' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3645261121"}]: dispatch 2024-04-30T09:32:35.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:35 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3645261121"}]: dispatch 2024-04-30T09:32:35.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:35 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/753133130"}]': finished 2024-04-30T09:32:35.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:35 smithi026 ceph-mon[27123]: osdmap e81: 8 total, 8 up, 8 in 2024-04-30T09:32:35.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:35 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/844025147' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3645261121"}]: dispatch 2024-04-30T09:32:35.798 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:35 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3645261121"}]: dispatch 2024-04-30T09:32:35.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:35 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/753133130"}]': finished 2024-04-30T09:32:35.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:35 smithi026 ceph-mon[22362]: osdmap e81: 8 total, 8 up, 8 in 2024-04-30T09:32:35.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:35 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/844025147' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3645261121"}]: dispatch 2024-04-30T09:32:35.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:35 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3645261121"}]: dispatch 2024-04-30T09:32:36.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 511 B/s wr, 5 op/s; 0 B/s, 0 objects/s recovering 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3645261121"}]': finished 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: osdmap e82: 8 total, 8 up, 8 in 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1682345591' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2396108168"}]: dispatch 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: Adding iSCSI gateway http://:@172.21.15.26:5000 to Dashboard 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: Checking dashboard <-> RGW credentials 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 511 B/s wr, 5 op/s; 0 B/s, 0 objects/s recovering 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3645261121"}]': finished 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: osdmap e82: 8 total, 8 up, 8 in 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1682345591' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2396108168"}]: dispatch 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: Adding iSCSI gateway http://:@172.21.15.26:5000 to Dashboard 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:36.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:36.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: Checking dashboard <-> RGW credentials 2024-04-30T09:32:36.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:36 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 511 B/s wr, 5 op/s; 0 B/s, 0 objects/s recovering 2024-04-30T09:32:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3645261121"}]': finished 2024-04-30T09:32:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: osdmap e82: 8 total, 8 up, 8 in 2024-04-30T09:32:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1682345591' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2396108168"}]: dispatch 2024-04-30T09:32:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:32:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: Adding iSCSI gateway http://:@172.21.15.26:5000 to Dashboard 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-add", "name": "smithi026"}]: dispatch 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: Checking dashboard <-> RGW credentials 2024-04-30T09:32:36.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:36 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:32:38.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:37 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1682345591' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2396108168"}]': finished 2024-04-30T09:32:38.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:37 smithi026 ceph-mon[22362]: osdmap e83: 8 total, 8 up, 8 in 2024-04-30T09:32:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:37 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/814250949' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3683337897"}]: dispatch 2024-04-30T09:32:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:37 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1682345591' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2396108168"}]': finished 2024-04-30T09:32:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:37 smithi026 ceph-mon[27123]: osdmap e83: 8 total, 8 up, 8 in 2024-04-30T09:32:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:37 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/814250949' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3683337897"}]: dispatch 2024-04-30T09:32:38.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:37 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1682345591' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2396108168"}]': finished 2024-04-30T09:32:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:37 smithi046 ceph-mon[23181]: osdmap e83: 8 total, 8 up, 8 in 2024-04-30T09:32:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:37 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/814250949' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3683337897"}]: dispatch 2024-04-30T09:32:39.135 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:38 smithi046 ceph-mon[23181]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:39.135 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:38 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/814250949' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3683337897"}]': finished 2024-04-30T09:32:39.135 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:38 smithi046 ceph-mon[23181]: osdmap e84: 8 total, 8 up, 8 in 2024-04-30T09:32:39.135 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:38 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1240736117' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/239085219"}]: dispatch 2024-04-30T09:32:39.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:38 smithi026 ceph-mon[27123]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:39.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:38 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/814250949' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3683337897"}]': finished 2024-04-30T09:32:39.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:38 smithi026 ceph-mon[27123]: osdmap e84: 8 total, 8 up, 8 in 2024-04-30T09:32:39.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:38 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1240736117' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/239085219"}]: dispatch 2024-04-30T09:32:39.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:38 smithi026 ceph-mon[22362]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 61 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:39.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:38 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/814250949' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3683337897"}]': finished 2024-04-30T09:32:39.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:38 smithi026 ceph-mon[22362]: osdmap e84: 8 total, 8 up, 8 in 2024-04-30T09:32:39.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:38 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1240736117' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/239085219"}]: dispatch 2024-04-30T09:32:40.147 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:39 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1240736117' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/239085219"}]': finished 2024-04-30T09:32:40.147 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:39 smithi046 ceph-mon[23181]: osdmap e85: 8 total, 8 up, 8 in 2024-04-30T09:32:40.147 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:39 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1692376766' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1844994599"}]: dispatch 2024-04-30T09:32:40.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:39 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1240736117' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/239085219"}]': finished 2024-04-30T09:32:40.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:39 smithi026 ceph-mon[27123]: osdmap e85: 8 total, 8 up, 8 in 2024-04-30T09:32:40.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:39 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1692376766' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1844994599"}]: dispatch 2024-04-30T09:32:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:39 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1240736117' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/239085219"}]': finished 2024-04-30T09:32:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:39 smithi026 ceph-mon[22362]: osdmap e85: 8 total, 8 up, 8 in 2024-04-30T09:32:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:39 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1692376766' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1844994599"}]: dispatch 2024-04-30T09:32:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[27123]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-30T09:32:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1692376766' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1844994599"}]': finished 2024-04-30T09:32:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[27123]: osdmap e86: 8 total, 8 up, 8 in 2024-04-30T09:32:41.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/367335851' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3072786577"}]: dispatch 2024-04-30T09:32:41.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3072786577"}]: dispatch 2024-04-30T09:32:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[22362]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-30T09:32:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1692376766' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1844994599"}]': finished 2024-04-30T09:32:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[22362]: osdmap e86: 8 total, 8 up, 8 in 2024-04-30T09:32:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/367335851' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3072786577"}]: dispatch 2024-04-30T09:32:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:40 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3072786577"}]: dispatch 2024-04-30T09:32:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:40 smithi046 ceph-mon[23181]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-30T09:32:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:40 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1692376766' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/1844994599"}]': finished 2024-04-30T09:32:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:40 smithi046 ceph-mon[23181]: osdmap e86: 8 total, 8 up, 8 in 2024-04-30T09:32:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:40 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/367335851' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3072786577"}]: dispatch 2024-04-30T09:32:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:40 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3072786577"}]: dispatch 2024-04-30T09:32:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:41 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3072786577"}]': finished 2024-04-30T09:32:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:41 smithi026 ceph-mon[27123]: osdmap e87: 8 total, 8 up, 8 in 2024-04-30T09:32:42.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:41 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1624873620' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/514779481"}]: dispatch 2024-04-30T09:32:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:41 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3072786577"}]': finished 2024-04-30T09:32:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:41 smithi026 ceph-mon[22362]: osdmap e87: 8 total, 8 up, 8 in 2024-04-30T09:32:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:41 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1624873620' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/514779481"}]: dispatch 2024-04-30T09:32:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:41 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3072786577"}]': finished 2024-04-30T09:32:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:41 smithi046 ceph-mon[23181]: osdmap e87: 8 total, 8 up, 8 in 2024-04-30T09:32:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:41 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1624873620' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/514779481"}]: dispatch 2024-04-30T09:32:42.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:42] "GET /metrics HTTP/1.1" 200 207872 "" "Prometheus/2.33.4" 2024-04-30T09:32:43.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[27123]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-30T09:32:43.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1624873620' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/514779481"}]': finished 2024-04-30T09:32:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[27123]: osdmap e88: 8 total, 8 up, 8 in 2024-04-30T09:32:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/448801475' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/326280148"}]: dispatch 2024-04-30T09:32:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/326280148"}]: dispatch 2024-04-30T09:32:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[22362]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-30T09:32:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1624873620' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/514779481"}]': finished 2024-04-30T09:32:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[22362]: osdmap e88: 8 total, 8 up, 8 in 2024-04-30T09:32:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/448801475' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/326280148"}]: dispatch 2024-04-30T09:32:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:42 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/326280148"}]: dispatch 2024-04-30T09:32:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:42 smithi046 ceph-mon[23181]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 62 MiB used, 715 GiB / 715 GiB avail; 9.2 KiB/s rd, 9 op/s 2024-04-30T09:32:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:42 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1624873620' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/514779481"}]': finished 2024-04-30T09:32:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:42 smithi046 ceph-mon[23181]: osdmap e88: 8 total, 8 up, 8 in 2024-04-30T09:32:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:42 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/448801475' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/326280148"}]: dispatch 2024-04-30T09:32:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:42 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/326280148"}]: dispatch 2024-04-30T09:32:44.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:32:44.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/326280148"}]': finished 2024-04-30T09:32:44.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[27123]: osdmap e89: 8 total, 8 up, 8 in 2024-04-30T09:32:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2225546082' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2113986196"}]: dispatch 2024-04-30T09:32:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2113986196"}]: dispatch 2024-04-30T09:32:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:32:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/326280148"}]': finished 2024-04-30T09:32:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[22362]: osdmap e89: 8 total, 8 up, 8 in 2024-04-30T09:32:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2225546082' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2113986196"}]: dispatch 2024-04-30T09:32:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:43 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2113986196"}]: dispatch 2024-04-30T09:32:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:43 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:32:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:43 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/326280148"}]': finished 2024-04-30T09:32:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:43 smithi046 ceph-mon[23181]: osdmap e89: 8 total, 8 up, 8 in 2024-04-30T09:32:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:43 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2225546082' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2113986196"}]: dispatch 2024-04-30T09:32:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:43 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2113986196"}]: dispatch 2024-04-30T09:32:45.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[27123]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:45.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2113986196"}]': finished 2024-04-30T09:32:45.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[27123]: osdmap e90: 8 total, 8 up, 8 in 2024-04-30T09:32:45.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3149690585' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/733429943"}]: dispatch 2024-04-30T09:32:45.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/733429943"}]: dispatch 2024-04-30T09:32:45.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[22362]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:45.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2113986196"}]': finished 2024-04-30T09:32:45.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[22362]: osdmap e90: 8 total, 8 up, 8 in 2024-04-30T09:32:45.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3149690585' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/733429943"}]: dispatch 2024-04-30T09:32:45.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:44 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/733429943"}]: dispatch 2024-04-30T09:32:45.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:32:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:32:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:44 smithi046 ceph-mon[23181]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:44 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2113986196"}]': finished 2024-04-30T09:32:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:44 smithi046 ceph-mon[23181]: osdmap e90: 8 total, 8 up, 8 in 2024-04-30T09:32:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:44 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3149690585' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/733429943"}]: dispatch 2024-04-30T09:32:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:44 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/733429943"}]: dispatch 2024-04-30T09:32:46.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:45 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/733429943"}]': finished 2024-04-30T09:32:46.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:45 smithi026 ceph-mon[27123]: osdmap e91: 8 total, 8 up, 8 in 2024-04-30T09:32:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:45 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/557171892' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/733429943"}]: dispatch 2024-04-30T09:32:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:45 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/733429943"}]: dispatch 2024-04-30T09:32:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:45 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/733429943"}]': finished 2024-04-30T09:32:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:45 smithi026 ceph-mon[22362]: osdmap e91: 8 total, 8 up, 8 in 2024-04-30T09:32:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:45 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/557171892' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/733429943"}]: dispatch 2024-04-30T09:32:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:45 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/733429943"}]: dispatch 2024-04-30T09:32:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:45 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/733429943"}]': finished 2024-04-30T09:32:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:45 smithi046 ceph-mon[23181]: osdmap e91: 8 total, 8 up, 8 in 2024-04-30T09:32:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:45 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/557171892' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/733429943"}]: dispatch 2024-04-30T09:32:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:45 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/733429943"}]: dispatch 2024-04-30T09:32:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[27123]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/733429943"}]': finished 2024-04-30T09:32:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[27123]: osdmap e92: 8 total, 8 up, 8 in 2024-04-30T09:32:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2722635693' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2396108168"}]: dispatch 2024-04-30T09:32:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2396108168"}]: dispatch 2024-04-30T09:32:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[22362]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/733429943"}]': finished 2024-04-30T09:32:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[22362]: osdmap e92: 8 total, 8 up, 8 in 2024-04-30T09:32:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2722635693' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2396108168"}]: dispatch 2024-04-30T09:32:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:46 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2396108168"}]: dispatch 2024-04-30T09:32:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:46 smithi046 ceph-mon[23181]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:46 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/733429943"}]': finished 2024-04-30T09:32:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:46 smithi046 ceph-mon[23181]: osdmap e92: 8 total, 8 up, 8 in 2024-04-30T09:32:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:46 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2722635693' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2396108168"}]: dispatch 2024-04-30T09:32:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:46 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2396108168"}]: dispatch 2024-04-30T09:32:48.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:47 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2396108168"}]': finished 2024-04-30T09:32:48.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:47 smithi026 ceph-mon[27123]: osdmap e93: 8 total, 8 up, 8 in 2024-04-30T09:32:48.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:47 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3901190785' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1844994599"}]: dispatch 2024-04-30T09:32:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:47 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2396108168"}]': finished 2024-04-30T09:32:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:47 smithi026 ceph-mon[22362]: osdmap e93: 8 total, 8 up, 8 in 2024-04-30T09:32:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:47 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3901190785' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1844994599"}]: dispatch 2024-04-30T09:32:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:47 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2396108168"}]': finished 2024-04-30T09:32:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:47 smithi046 ceph-mon[23181]: osdmap e93: 8 total, 8 up, 8 in 2024-04-30T09:32:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:47 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3901190785' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1844994599"}]: dispatch 2024-04-30T09:32:49.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:48 smithi026 ceph-mon[27123]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:48 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3901190785' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1844994599"}]': finished 2024-04-30T09:32:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:48 smithi026 ceph-mon[27123]: osdmap e94: 8 total, 8 up, 8 in 2024-04-30T09:32:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:48 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1709981825' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4095183837"}]: dispatch 2024-04-30T09:32:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:48 smithi026 ceph-mon[22362]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:48 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3901190785' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1844994599"}]': finished 2024-04-30T09:32:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:48 smithi026 ceph-mon[22362]: osdmap e94: 8 total, 8 up, 8 in 2024-04-30T09:32:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:48 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1709981825' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4095183837"}]: dispatch 2024-04-30T09:32:49.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:48 smithi046 ceph-mon[23181]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:32:49.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:48 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3901190785' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/1844994599"}]': finished 2024-04-30T09:32:49.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:48 smithi046 ceph-mon[23181]: osdmap e94: 8 total, 8 up, 8 in 2024-04-30T09:32:49.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:48 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1709981825' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4095183837"}]: dispatch 2024-04-30T09:32:50.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:49 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1709981825' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4095183837"}]': finished 2024-04-30T09:32:50.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:49 smithi026 ceph-mon[27123]: osdmap e95: 8 total, 8 up, 8 in 2024-04-30T09:32:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:49 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1709981825' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4095183837"}]': finished 2024-04-30T09:32:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:49 smithi026 ceph-mon[22362]: osdmap e95: 8 total, 8 up, 8 in 2024-04-30T09:32:50.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:49 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1709981825' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4095183837"}]': finished 2024-04-30T09:32:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:49 smithi046 ceph-mon[23181]: osdmap e95: 8 total, 8 up, 8 in 2024-04-30T09:32:51.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:50 smithi026 ceph-mon[27123]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:51.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:50 smithi026 ceph-mon[22362]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:51.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:50 smithi046 ceph-mon[23181]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:52.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:32:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:52] "GET /metrics HTTP/1.1" 200 207990 "" "Prometheus/2.33.4" 2024-04-30T09:32:53.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:52 smithi026 ceph-mon[22362]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-04-30T09:32:53.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:52 smithi026 ceph-mon[27123]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-04-30T09:32:53.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:52 smithi046 ceph-mon[23181]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 63 MiB used, 715 GiB / 715 GiB avail; 900 B/s rd, 0 op/s 2024-04-30T09:32:54.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:53 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:32:54.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:53 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:32:54.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:53 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:32:55.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:54 smithi026 ceph-mon[22362]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-30T09:32:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:54 smithi026 ceph-mon[27123]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-30T09:32:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:54 smithi046 ceph-mon[23181]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1 op/s 2024-04-30T09:32:55.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:32:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:32:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:32:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:56 smithi026 ceph-mon[27123]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:56 smithi026 ceph-mon[22362]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:56 smithi046 ceph-mon[23181]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:32:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:32:58 smithi026 ceph-mon[27123]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:32:59.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:32:58 smithi026 ceph-mon[22362]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:32:59.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:32:58 smithi046 ceph-mon[23181]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:33:01.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:00 smithi026 ceph-mon[27123]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 960 B/s rd, 0 op/s 2024-04-30T09:33:01.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:00 smithi026 ceph-mon[22362]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 960 B/s rd, 0 op/s 2024-04-30T09:33:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:00 smithi046 ceph-mon[23181]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 960 B/s rd, 0 op/s 2024-04-30T09:33:02.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:33:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:02] "GET /metrics HTTP/1.1" 200 207974 "" "Prometheus/2.33.4" 2024-04-30T09:33:03.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:02 smithi026 ceph-mon[27123]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:03.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:02 smithi026 ceph-mon[22362]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:02 smithi046 ceph-mon[23181]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:03 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:04.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:03 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:03 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:05.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:04 smithi026 ceph-mon[27123]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:04 smithi026 ceph-mon[22362]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:04 smithi046 ceph-mon[23181]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:05.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:33:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:33:07.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:06 smithi026 ceph-mon[27123]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:07.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:06 smithi026 ceph-mon[22362]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:06 smithi046 ceph-mon[23181]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:09.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:08 smithi026 ceph-mon[27123]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:09.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:08 smithi026 ceph-mon[22362]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:09.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:08 smithi046 ceph-mon[23181]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:11.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:10 smithi026 ceph-mon[27123]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:11.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:10 smithi026 ceph-mon[22362]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:11.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:10 smithi046 ceph-mon[23181]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:12.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:33:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:12] "GET /metrics HTTP/1.1" 200 207974 "" "Prometheus/2.33.4" 2024-04-30T09:33:13.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:12 smithi026 ceph-mon[22362]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:13.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:12 smithi026 ceph-mon[27123]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:13.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:12 smithi046 ceph-mon[23181]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:14.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:13 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:14.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:13 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:13 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:15.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:14 smithi026 ceph-mon[22362]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:15.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:14 smithi026 ceph-mon[27123]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:15.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:14 smithi046 ceph-mon[23181]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:15.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:33:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:33:17.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:16 smithi026 ceph-mon[22362]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:17.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:16 smithi026 ceph-mon[27123]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:16 smithi046 ceph-mon[23181]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:19.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:18 smithi026 ceph-mon[22362]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:18 smithi026 ceph-mon[27123]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:18 smithi046 ceph-mon[23181]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:21.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:20 smithi026 ceph-mon[22362]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:21.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:20 smithi026 ceph-mon[27123]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:21.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:20 smithi046 ceph-mon[23181]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:22.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:33:22 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:22] "GET /metrics HTTP/1.1" 200 207926 "" "Prometheus/2.33.4" 2024-04-30T09:33:23.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:22 smithi026 ceph-mon[27123]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:23.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:22 smithi026 ceph-mon[22362]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:23.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:22 smithi046 ceph-mon[23181]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:24.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:23 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:24.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:23 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:24.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:23 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:24 smithi026 ceph-mon[27123]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:24 smithi026 ceph-mon[22362]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:25.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:24 smithi046 ceph-mon[23181]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:25.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:33:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:33:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:26 smithi046 ceph-mon[23181]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:27.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:26 smithi026 ceph-mon[27123]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:27.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:26 smithi026 ceph-mon[22362]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:33:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:33:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:33:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:33:28.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:33:28.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:33:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:33:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:33:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:33:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:33:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:33:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:33:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:28 smithi046 ceph-mon[23181]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:29.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:28 smithi026 ceph-mon[22362]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:29.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:28 smithi026 ceph-mon[27123]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:30 smithi046 ceph-mon[23181]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:31.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:30 smithi026 ceph-mon[22362]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:31.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:30 smithi026 ceph-mon[27123]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:32.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:33:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:32] "GET /metrics HTTP/1.1" 200 207883 "" "Prometheus/2.33.4" 2024-04-30T09:33:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:32 smithi046 ceph-mon[23181]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:33.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:32 smithi026 ceph-mon[22362]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:33.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:32 smithi026 ceph-mon[27123]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:33 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:34.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:33 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:33 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:34 smithi046 ceph-mon[23181]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:35.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:33:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:33:35.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:34 smithi026 ceph-mon[27123]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:35.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:34 smithi026 ceph-mon[22362]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:36 smithi046 ceph-mon[23181]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:36 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:33:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:36 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:33:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:36 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:33:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:36 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:33:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[27123]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:33:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:33:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:33:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:33:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[22362]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:33:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:33:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:33:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:36 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:33:39.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:38 smithi046 ceph-mon[23181]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:39.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:38 smithi026 ceph-mon[27123]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:39.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:38 smithi026 ceph-mon[22362]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:40 smithi046 ceph-mon[23181]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:41.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:40 smithi026 ceph-mon[27123]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:41.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:40 smithi026 ceph-mon[22362]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:42.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:33:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:42] "GET /metrics HTTP/1.1" 200 207883 "" "Prometheus/2.33.4" 2024-04-30T09:33:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:42 smithi046 ceph-mon[23181]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:43.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:42 smithi026 ceph-mon[22362]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:43.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:42 smithi026 ceph-mon[27123]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:43 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:44.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:43 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:43 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:44 smithi046 ceph-mon[23181]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:45.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:33:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:33:45.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:44 smithi026 ceph-mon[27123]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:45.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:44 smithi026 ceph-mon[22362]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:47.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:46 smithi026 ceph-mon[27123]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:47.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:46 smithi026 ceph-mon[22362]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:46 smithi046 ceph-mon[23181]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:48 smithi026 ceph-mon[27123]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:49.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:48 smithi026 ceph-mon[22362]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:48 smithi046 ceph-mon[23181]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:51.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:50 smithi026 ceph-mon[27123]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:50 smithi026 ceph-mon[22362]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:50 smithi046 ceph-mon[23181]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:52.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:33:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:52] "GET /metrics HTTP/1.1" 200 207895 "" "Prometheus/2.33.4" 2024-04-30T09:33:53.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:52 smithi026 ceph-mon[27123]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:52 smithi026 ceph-mon[22362]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:52 smithi046 ceph-mon[23181]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:54.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:53 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:54.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:53 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:54.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:53 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:33:55.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:54 smithi026 ceph-mon[27123]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:55.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:54 smithi026 ceph-mon[22362]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:54 smithi046 ceph-mon[23181]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:33:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:33:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:33:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:33:57.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:56 smithi026 ceph-mon[27123]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:57.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:56 smithi026 ceph-mon[22362]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:56 smithi046 ceph-mon[23181]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:59.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:33:59 smithi026 ceph-mon[27123]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:59.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:33:59 smithi026 ceph-mon[22362]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:33:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:33:59 smithi046 ceph-mon[23181]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:01.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:01 smithi026 ceph-mon[27123]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:01.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:01 smithi026 ceph-mon[22362]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:01 smithi046 ceph-mon[23181]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:02.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:34:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:02] "GET /metrics HTTP/1.1" 200 207877 "" "Prometheus/2.33.4" 2024-04-30T09:34:03.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:03 smithi026 ceph-mon[27123]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:03.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:03 smithi026 ceph-mon[22362]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:03 smithi046 ceph-mon[23181]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:04 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:04.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:04 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:04.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:04 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:05.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:05 smithi026 ceph-mon[22362]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:05.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:05 smithi026 ceph-mon[27123]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:05 smithi046 ceph-mon[23181]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:34:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:34:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:07 smithi026 ceph-mon[27123]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:07 smithi026 ceph-mon[22362]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:07 smithi046 ceph-mon[23181]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:09.286 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:09 smithi046 ceph-mon[23181]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:09.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:09 smithi026 ceph-mon[27123]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:09.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:09 smithi026 ceph-mon[22362]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:11.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:11 smithi026 ceph-mon[27123]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:11.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:11 smithi026 ceph-mon[22362]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:11 smithi046 ceph-mon[23181]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:12.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:34:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:12] "GET /metrics HTTP/1.1" 200 207877 "" "Prometheus/2.33.4" 2024-04-30T09:34:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:13 smithi026 ceph-mon[27123]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:13 smithi026 ceph-mon[22362]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:13 smithi046 ceph-mon[23181]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:14 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:14 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:14 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:15.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:15 smithi026 ceph-mon[27123]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:15.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:15 smithi026 ceph-mon[22362]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:15 smithi046 ceph-mon[23181]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:34:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:34:17.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:17 smithi026 ceph-mon[27123]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:17 smithi026 ceph-mon[22362]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:17 smithi046 ceph-mon[23181]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:19.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:19 smithi026 ceph-mon[27123]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:19.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:19 smithi026 ceph-mon[22362]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:19 smithi046 ceph-mon[23181]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:21.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:21 smithi026 ceph-mon[27123]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:21.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:21 smithi026 ceph-mon[22362]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:21.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:21 smithi046 ceph-mon[23181]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:22.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:34:22 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:22] "GET /metrics HTTP/1.1" 200 207904 "" "Prometheus/2.33.4" 2024-04-30T09:34:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:23 smithi026 ceph-mon[27123]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:23 smithi026 ceph-mon[22362]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:23 smithi046 ceph-mon[23181]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:24.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:24 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:24 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:24 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:25.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:25 smithi026 ceph-mon[27123]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:25.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:25 smithi026 ceph-mon[22362]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:25 smithi046 ceph-mon[23181]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:34:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:34:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:27 smithi026 ceph-mon[27123]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:27 smithi026 ceph-mon[22362]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:27 smithi046 ceph-mon[23181]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:34:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:34:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:34:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:34:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:34:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:34:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:34:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:34:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:34:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:34:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:34:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:34:29.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:29 smithi026 ceph-mon[27123]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:29.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:29 smithi026 ceph-mon[22362]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:29 smithi046 ceph-mon[23181]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:31.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:31 smithi026 ceph-mon[27123]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:31.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:31 smithi026 ceph-mon[22362]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:31 smithi046 ceph-mon[23181]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:32.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:34:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:32] "GET /metrics HTTP/1.1" 200 207898 "" "Prometheus/2.33.4" 2024-04-30T09:34:33.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:33 smithi026 ceph-mon[27123]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:33.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:33 smithi026 ceph-mon[22362]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:33 smithi046 ceph-mon[23181]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:34.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:34 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:34 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:34 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:35.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:35 smithi026 ceph-mon[22362]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:35.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:35 smithi026 ceph-mon[27123]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:35.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:35 smithi046 ceph-mon[23181]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:34:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:34:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[27123]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:34:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:34:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:34:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:34:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[22362]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:34:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:34:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:34:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:37 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:34:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:37 smithi046 ceph-mon[23181]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:37 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:34:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:37 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:34:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:37 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:34:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:37 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:34:39.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:39 smithi026 ceph-mon[27123]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:39.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:39 smithi026 ceph-mon[22362]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:39 smithi046 ceph-mon[23181]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:41.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:41 smithi026 ceph-mon[22362]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:41.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:41 smithi026 ceph-mon[27123]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:41.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:41 smithi046 ceph-mon[23181]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:42.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:34:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:42] "GET /metrics HTTP/1.1" 200 207898 "" "Prometheus/2.33.4" 2024-04-30T09:34:43.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:43 smithi026 ceph-mon[22362]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:34:43.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:43 smithi026 ceph-mon[27123]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:34:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:43 smithi046 ceph-mon[23181]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:34:44.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:44 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:44 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:44 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:45.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:45 smithi026 ceph-mon[22362]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:45.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:45 smithi026 ceph-mon[27123]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:45.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:45 smithi046 ceph-mon[23181]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:45.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:34:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:34:47.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:47 smithi026 ceph-mon[27123]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:34:47.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:47 smithi026 ceph-mon[22362]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:34:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:47 smithi046 ceph-mon[23181]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:34:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:49 smithi026 ceph-mon[27123]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:34:49.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:49 smithi026 ceph-mon[22362]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:34:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:49 smithi046 ceph-mon[23181]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:34:51.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:51 smithi026 ceph-mon[27123]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:51 smithi026 ceph-mon[22362]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:51 smithi046 ceph-mon[23181]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:52.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:34:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:52] "GET /metrics HTTP/1.1" 200 207885 "" "Prometheus/2.33.4" 2024-04-30T09:34:53.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:53 smithi026 ceph-mon[22362]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:53 smithi026 ceph-mon[27123]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:53 smithi046 ceph-mon[23181]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:54.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:54 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:54 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:54.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:54 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:34:55.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:55 smithi026 ceph-mon[22362]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:55.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:55 smithi026 ceph-mon[27123]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:55 smithi046 ceph-mon[23181]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:34:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:34:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:34:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:34:57.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:57 smithi026 ceph-mon[22362]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:57.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:57 smithi026 ceph-mon[27123]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:57 smithi046 ceph-mon[23181]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:59.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:34:59 smithi026 ceph-mon[22362]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:34:59 smithi026 ceph-mon[27123]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:34:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:34:59 smithi046 ceph-mon[23181]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:01.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:01 smithi026 ceph-mon[22362]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:01.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:01 smithi026 ceph-mon[27123]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:01 smithi046 ceph-mon[23181]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:02.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:35:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:02] "GET /metrics HTTP/1.1" 200 207838 "" "Prometheus/2.33.4" 2024-04-30T09:35:03.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:03 smithi026 ceph-mon[22362]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:03.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:03 smithi026 ceph-mon[27123]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:03 smithi046 ceph-mon[23181]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:04 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:04 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:04.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:04 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:05.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:05 smithi026 ceph-mon[27123]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:05 smithi026 ceph-mon[22362]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:05 smithi046 ceph-mon[23181]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:35:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:35:07.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:07 smithi026 ceph-mon[22362]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:07 smithi026 ceph-mon[27123]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:07 smithi046 ceph-mon[23181]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:09.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:09 smithi026 ceph-mon[27123]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:09.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:09 smithi026 ceph-mon[22362]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:09 smithi046 ceph-mon[23181]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:11.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:11 smithi026 ceph-mon[27123]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:11.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:11 smithi026 ceph-mon[22362]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:11 smithi046 ceph-mon[23181]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:12.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:35:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:12] "GET /metrics HTTP/1.1" 200 207838 "" "Prometheus/2.33.4" 2024-04-30T09:35:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:13 smithi026 ceph-mon[27123]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:13 smithi026 ceph-mon[22362]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:13 smithi046 ceph-mon[23181]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:14 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:14 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:14 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:15 smithi026 ceph-mon[27123]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:15.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:15 smithi026 ceph-mon[22362]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:15 smithi046 ceph-mon[23181]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:35:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:35:17.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:17 smithi026 ceph-mon[22362]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:17 smithi026 ceph-mon[27123]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:17 smithi046 ceph-mon[23181]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:19.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:19 smithi026 ceph-mon[22362]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:19 smithi026 ceph-mon[27123]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:19 smithi046 ceph-mon[23181]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:21.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:21 smithi026 ceph-mon[22362]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:21 smithi026 ceph-mon[27123]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:21 smithi046 ceph-mon[23181]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:22.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:35:22 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:22] "GET /metrics HTTP/1.1" 200 207834 "" "Prometheus/2.33.4" 2024-04-30T09:35:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:23 smithi026 ceph-mon[27123]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:23.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:23 smithi026 ceph-mon[22362]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:23 smithi046 ceph-mon[23181]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:24.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:24 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:24 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:24 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:25.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:25 smithi026 ceph-mon[27123]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:25.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:25 smithi026 ceph-mon[22362]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:25 smithi046 ceph-mon[23181]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:35:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:35:26.882 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-30T09:35:27.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:27 smithi026 ceph-mon[27123]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:27.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:27 smithi026 ceph-mon[22362]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:27 smithi046 ceph-mon[23181]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:28.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:35:28.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:35:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:28 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:35:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:28 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:35:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:35:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:35:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:28 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:35:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:28 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:35:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:35:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:35:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:28 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:35:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:28 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:35:29.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:29 smithi026 ceph-mon[22362]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:29.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:29 smithi026 ceph-mon[27123]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:29 smithi046 ceph-mon[23181]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:29.490 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-30T09:35:31.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:31 smithi026 ceph-mon[22362]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:31.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:31 smithi026 ceph-mon[27123]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:31 smithi046 ceph-mon[23181]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:32.014 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-30T09:35:32.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:35:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:32] "GET /metrics HTTP/1.1" 200 207831 "" "Prometheus/2.33.4" 2024-04-30T09:35:33.261 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:33 smithi026 ceph-mon[27123]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:33.262 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:33 smithi026 ceph-mon[22362]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:33 smithi046 ceph-mon[23181]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:34.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:34 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:34.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:34 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:34 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:34.587 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T09:35:35.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:35 smithi026 ceph-mon[22362]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:35.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:35 smithi026 ceph-mon[27123]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:35.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:35 smithi046 ceph-mon[23181]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:35:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:35:36.561 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:35:36.561 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 3m ago 4m 15.4M - ba2b418f427c 7b0e3dcf393a 2024-04-30T09:35:36.561 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (3m) 3m ago 3m 47.5M - 8.3.5 dad864ee21e9 9ecc79f0568a 2024-04-30T09:35:36.561 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 running (3m) 3m ago 3m 68.4M - 3.5 e1d6a67b021e 68f3038ef8d6 2024-04-30T09:35:36.561 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 2b8d2d89eb23 2024-04-30T09:35:36.561 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (10m) 3m ago 10m 455M - 17.2.0 e1d6a67b021e 780f8f988439 2024-04-30T09:35:36.561 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (10m) 3m ago 10m 49.1M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:35:36.561 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (8m) 3m ago 8m 38.1M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (9m) 3m ago 9m 42.6M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (4m) 3m ago 4m 17.2M - 1dbe0e931976 d712368b56f8 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (4m) 3m ago 4m 16.2M - 1dbe0e931976 727cc4da58b3 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (8m) 3m ago 8m 47.3M 4017M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (8m) 3m ago 7m 46.6M 4017M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (7m) 3m ago 7m 46.7M 4017M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (7m) 3m ago 7m 44.5M 4017M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (6m) 3m ago 6m 48.4M 4273M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (6m) 3m ago 6m 44.2M 4273M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (6m) 3m ago 5m 43.6M 4273M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (5m) 3m ago 5m 45.9M 4273M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (3m) 3m ago 4m 44.1M - 514e6a882f6e f54f9679bceb 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (3m) 3m ago 3m 69.1M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:35:36.562 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (3m) 3m ago 3m 69.0M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:35:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:37 smithi026 ceph-mon[27123]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:37 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:35:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:37 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:35:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:37 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:35:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:37 smithi026 ceph-mon[22362]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:37 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:35:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:37 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:35:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:37 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:35:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:37 smithi046 ceph-mon[23181]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:37 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:35:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:37 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:35:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:37 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:35:37.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions' 2024-04-30T09:35:38.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:38 smithi026 ceph-mon[27123]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:38.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:38 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:35:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:38 smithi026 ceph-mon[22362]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:38 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:35:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:38 smithi046 ceph-mon[23181]: from='client.14997 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:38 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:35:39.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:39 smithi026 ceph-mon[22362]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:39.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:39 smithi026 ceph-mon[27123]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:39 smithi046 ceph-mon[23181]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:39.595 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:35:39.596 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:35:39.597 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 15 2024-04-30T09:35:39.597 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:35:39.597 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:35:40.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph -s' 2024-04-30T09:35:40.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:40 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/720043185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:35:40.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:40 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/720043185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:35:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:40 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/720043185' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:35:41.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:41 smithi026 ceph-mon[27123]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:41.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:41 smithi026 ceph-mon[22362]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:41.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:41 smithi046 ceph-mon[23181]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:42.303 INFO:teuthology.orchestra.run.smithi026.stdout: cluster: 2024-04-30T09:35:42.303 INFO:teuthology.orchestra.run.smithi026.stdout: id: 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:35:42.303 INFO:teuthology.orchestra.run.smithi026.stdout: health: HEALTH_OK 2024-04-30T09:35:42.303 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:35:42.303 INFO:teuthology.orchestra.run.smithi026.stdout: services: 2024-04-30T09:35:42.303 INFO:teuthology.orchestra.run.smithi026.stdout: mon: 3 daemons, quorum a,c,b (age 8m) 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: mgr: y(active, since 5m), standbys: x 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: osd: 8 osds: 8 up (since 5m), 8 in (since 5m) 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: data: 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: pools: 6 pools, 161 pgs 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: objects: 209 objects, 585 KiB 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: pgs: 161 active+clean 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: io: 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-30T09:35:42.304 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:35:42.622 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:35:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:42] "GET /metrics HTTP/1.1" 200 207831 "" "Prometheus/2.33.4" 2024-04-30T09:35:42.964 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ls' 2024-04-30T09:35:43.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:43 smithi026 ceph-mon[27123]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:43.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:43 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1094314314' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:35:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:43 smithi046 ceph-mon[23181]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:43 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1094314314' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:35:43.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:43 smithi026 ceph-mon[22362]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:43.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:43 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1094314314' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:35:44.263 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:44 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:44.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:44 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:44 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:44.973 INFO:teuthology.orchestra.run.smithi026.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-30T09:35:44.973 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager ?:9093,9094 1/1 3m ago 5m smithi026=a;count:1 2024-04-30T09:35:44.974 INFO:teuthology.orchestra.run.smithi026.stdout:grafana ?:3000 1/1 3m ago 5m smithi046=a;count:1 2024-04-30T09:35:44.974 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo 1/1 3m ago 3m count:1 2024-04-30T09:35:44.974 INFO:teuthology.orchestra.run.smithi026.stdout:mgr 2/2 3m ago 8m smithi026=y;smithi046=x;count:2 2024-04-30T09:35:44.974 INFO:teuthology.orchestra.run.smithi026.stdout:mon 3/3 3m ago 9m smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi046:172.21.15.46=b;count:3 2024-04-30T09:35:44.974 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter ?:9100 2/2 3m ago 5m smithi026=a;smithi046=b;count:2 2024-04-30T09:35:44.974 INFO:teuthology.orchestra.run.smithi026.stdout:osd 8 3m ago - 2024-04-30T09:35:44.974 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus ?:9095 1/1 3m ago 5m smithi046=a;count:1 2024-04-30T09:35:44.974 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo ?:8000 2/2 3m ago 3m count:2 2024-04-30T09:35:45.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:45 smithi026 ceph-mon[27123]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:45.303 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:45 smithi026 ceph-mon[22362]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:45.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:45 smithi046 ceph-mon[23181]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:45.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:35:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:35:45.735 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-30T09:35:46.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:46 smithi026 ceph-mon[27123]: from='client.24907 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:46.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:46 smithi026 ceph-mon[22362]: from='client.24907 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:46 smithi046 ceph-mon[23181]: from='client.24907 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:47 smithi046 ceph-mon[23181]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:47.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:47 smithi026 ceph-mon[22362]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:47.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:47 smithi026 ceph-mon[27123]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:48.033 INFO:teuthology.orchestra.run.smithi026.stdout:Scheduled to redeploy mgr.x on host 'smithi046' 2024-04-30T09:35:48.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:48 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4248192160' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-30T09:35:48.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:48 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:35:48.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:48 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:35:48.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:48 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4248192160' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-30T09:35:48.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:48 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:35:48.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:48 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:35:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:48 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4248192160' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-30T09:35:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:48 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:35:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:48 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:35:48.845 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps --refresh' 2024-04-30T09:35:49.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:49 smithi026 ceph-mon[27123]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:49.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:49 smithi026 ceph-mon[27123]: from='client.15027 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:49.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:49 smithi026 ceph-mon[27123]: Schedule redeploy daemon mgr.x 2024-04-30T09:35:49.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:49 smithi026 ceph-mon[22362]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:49.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:49 smithi026 ceph-mon[22362]: from='client.15027 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:49.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:49 smithi026 ceph-mon[22362]: Schedule redeploy daemon mgr.x 2024-04-30T09:35:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:49 smithi046 ceph-mon[23181]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:49 smithi046 ceph-mon[23181]: from='client.15027 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.x", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:49 smithi046 ceph-mon[23181]: Schedule redeploy daemon mgr.x 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 3m ago 4m 15.4M - ba2b418f427c 7b0e3dcf393a 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (4m) 3m ago 4m 47.5M - 8.3.5 dad864ee21e9 9ecc79f0568a 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 running (3m) 3m ago 3m 68.4M - 3.5 e1d6a67b021e 68f3038ef8d6 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443 running (8m) 3m ago 8m 410M - 17.2.0 e1d6a67b021e 2b8d2d89eb23 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (10m) 3m ago 10m 455M - 17.2.0 e1d6a67b021e 780f8f988439 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (10m) 3m ago 10m 49.1M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (9m) 3m ago 9m 38.1M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (9m) 3m ago 9m 42.6M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:35:51.217 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (5m) 3m ago 5m 17.2M - 1dbe0e931976 d712368b56f8 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (4m) 3m ago 4m 16.2M - 1dbe0e931976 727cc4da58b3 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (8m) 3m ago 8m 47.3M 4017M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (8m) 3m ago 8m 46.6M 4017M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (7m) 3m ago 7m 46.7M 4017M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (7m) 3m ago 7m 44.5M 4017M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (7m) 3m ago 7m 48.4M 4273M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (6m) 3m ago 6m 44.2M 4273M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (6m) 3m ago 6m 43.6M 4273M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (5m) 3m ago 5m 45.9M 4273M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (3m) 3m ago 5m 44.1M - 514e6a882f6e f54f9679bceb 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (3m) 3m ago 3m 69.1M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:35:51.218 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (3m) 3m ago 3m 69.0M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:35:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:51 smithi046 ceph-mon[23181]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:51.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:51 smithi026 ceph-mon[27123]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:51.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:51 smithi026 ceph-mon[22362]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:52.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'sleep 180' 2024-04-30T09:35:52.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:52 smithi026 ceph-mon[22362]: from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:52.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:52 smithi026 ceph-mon[27123]: from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:52.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:52 smithi046 ceph-mon[23181]: from='client.24925 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:35:52.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:35:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:52] "GET /metrics HTTP/1.1" 200 207837 "" "Prometheus/2.33.4" 2024-04-30T09:35:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:53 smithi046 ceph-mon[23181]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:53.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:53 smithi026 ceph-mon[27123]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:53.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:53 smithi026 ceph-mon[22362]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:54 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:54.502 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:54 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:54.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:54 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:35:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:55 smithi046 ceph-mon[23181]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:35:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:35:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:35:55.530 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:55 smithi026 ceph-mon[27123]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:55.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:55 smithi026 ceph-mon[22362]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:35:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:57 smithi046 ceph-mon[23181]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:57.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:57 smithi026 ceph-mon[27123]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:57.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:57 smithi026 ceph-mon[22362]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:35:59 smithi046 ceph-mon[23181]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:59.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:35:59 smithi026 ceph-mon[27123]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:35:59.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:35:59 smithi026 ceph-mon[22362]: pgmap v204: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:01.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:01 smithi026 ceph-mon[27123]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:01.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:01 smithi026 ceph-mon[22362]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:01.733 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:01 smithi046 ceph-mon[23181]: pgmap v205: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:02.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:02 smithi026 ceph-mon[27123]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:02.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:02 smithi026 ceph-mon[22362]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:02.505 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:36:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:02] "GET /metrics HTTP/1.1" 200 207850 "" "Prometheus/2.33.4" 2024-04-30T09:36:02.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:02 smithi046 ceph-mon[23181]: pgmap v206: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:03.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:03 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:03.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:03 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:03.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:03 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:04 smithi026 ceph-mon[27123]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:04 smithi026 ceph-mon[22362]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:04.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:04 smithi046 ceph-mon[23181]: pgmap v207: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:36:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:36:06.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:06 smithi026 ceph-mon[27123]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:06.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:06 smithi026 ceph-mon[22362]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:06.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:06 smithi046 ceph-mon[23181]: pgmap v208: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:08.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:08 smithi026 ceph-mon[27123]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:08.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:08 smithi026 ceph-mon[22362]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:08.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:08 smithi046 ceph-mon[23181]: pgmap v209: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:10.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:10 smithi026 ceph-mon[27123]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:10.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:10 smithi026 ceph-mon[22362]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:10.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:10 smithi046 ceph-mon[23181]: pgmap v210: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:12.689 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:36:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:12] "GET /metrics HTTP/1.1" 200 207850 "" "Prometheus/2.33.4" 2024-04-30T09:36:12.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:12 smithi026 ceph-mon[27123]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:12.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:12 smithi026 ceph-mon[22362]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:12.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:12 smithi046 ceph-mon[23181]: pgmap v211: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:13.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:13 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:13.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:13 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:13.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:13 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:14.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:14 smithi026 ceph-mon[27123]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:14 smithi026 ceph-mon[22362]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:14.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:14 smithi046 ceph-mon[23181]: pgmap v212: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:36:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:36:16.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:16 smithi046 ceph-mon[23181]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:17.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:16 smithi026 ceph-mon[27123]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:17.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:16 smithi026 ceph-mon[22362]: pgmap v213: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:18.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:18 smithi046 ceph-mon[23181]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:19.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:18 smithi026 ceph-mon[27123]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:19.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:18 smithi026 ceph-mon[22362]: pgmap v214: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:20.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:20 smithi046 ceph-mon[23181]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:20.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:20 smithi026 ceph-mon[27123]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:20.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:20 smithi026 ceph-mon[22362]: pgmap v215: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:22.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:36:22 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:22] "GET /metrics HTTP/1.1" 200 207862 "" "Prometheus/2.33.4" 2024-04-30T09:36:23.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:23 smithi046 ceph-mon[23181]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:23.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:23 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:23.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:23 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:36:23.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:23 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:36:23.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:23 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:23.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:23 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:23.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[27123]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:23.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:23.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:36:23.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:36:23.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:23.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[22362]: pgmap v216: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:36:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:36:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:23 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:24.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:24 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:24.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:24 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:24.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:24 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:25 smithi026 ceph-mon[27123]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:25 smithi026 ceph-mon[22362]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:25.280 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:25 smithi046 ceph-mon[23181]: pgmap v217: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:25.280 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:36:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:36:27.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[22362]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:36:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:36:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:36:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:36:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[22362]: Deploying daemon mgr.x on smithi046 2024-04-30T09:36:27.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[27123]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:27.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:27.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:27.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:27.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:36:27.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:36:27.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:36:27.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:36:27.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:26 smithi026 ceph-mon[27123]: Deploying daemon mgr.x on smithi046 2024-04-30T09:36:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:26 smithi046 ceph-mon[23181]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:26 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:26 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:26 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:36:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:26 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:36:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:26 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:36:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:26 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:36:27.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:26 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:36:27.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:26 smithi046 ceph-mon[23181]: Deploying daemon mgr.x on smithi046 2024-04-30T09:36:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:36:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:36:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:36:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:36:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:36:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:36:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:36:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:36:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:36:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:36:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:36:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:36:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:28 smithi026 ceph-mon[27123]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:29.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:28 smithi026 ceph-mon[22362]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:29.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:28 smithi046 ceph-mon[23181]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:31.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:30 smithi026 ceph-mon[27123]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:30 smithi026 ceph-mon[22362]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:31.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:30 smithi046 ceph-mon[23181]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:32.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:36:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:32] "GET /metrics HTTP/1.1" 200 207855 "" "Prometheus/2.33.4" 2024-04-30T09:36:33.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:32 smithi026 ceph-mon[27123]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:32 smithi026 ceph-mon[22362]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:33.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:32 smithi046 ceph-mon[23181]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:34.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:33 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:33 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:34.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:33 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:34 smithi026 ceph-mon[27123]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:34 smithi026 ceph-mon[22362]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:34 smithi046 ceph-mon[23181]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:35.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:36:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:36:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:36 smithi026 ceph-mon[27123]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:36 smithi026 ceph-mon[22362]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:37.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:36 smithi046 ceph-mon[23181]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:39.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:38 smithi026 ceph-mon[27123]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:39.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:38 smithi026 ceph-mon[22362]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:39.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:38 smithi046 ceph-mon[23181]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:40 smithi026 ceph-mon[27123]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:40 smithi026 ceph-mon[22362]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:41.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:40 smithi046 ceph-mon[23181]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:42.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:36:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:42] "GET /metrics HTTP/1.1" 200 207855 "" "Prometheus/2.33.4" 2024-04-30T09:36:43.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:42 smithi026 ceph-mon[22362]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:42 smithi026 ceph-mon[27123]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:43.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:42 smithi046 ceph-mon[23181]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:43.881 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:36:43 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:36:43.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:36:43.881 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:36:43 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:36:43.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:36:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:43 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:43 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:44.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:43 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:45.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:44 smithi026 ceph-mon[22362]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:45.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:44 smithi026 ceph-mon[27123]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:44 smithi046 ceph-mon[23181]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:45.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:36:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:36:47.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:46 smithi026 ceph-mon[22362]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:46 smithi026 ceph-mon[27123]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:46 smithi046 ceph-mon[23181]: pgmap v228: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:49.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:48 smithi026 ceph-mon[22362]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:48 smithi026 ceph-mon[27123]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:49.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:48 smithi046 ceph-mon[23181]: pgmap v229: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:51.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:50 smithi026 ceph-mon[27123]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:51.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:50 smithi026 ceph-mon[22362]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:50 smithi046 ceph-mon[23181]: pgmap v230: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:52.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:36:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:52] "GET /metrics HTTP/1.1" 200 207853 "" "Prometheus/2.33.4" 2024-04-30T09:36:53.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:52 smithi026 ceph-mon[27123]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:53.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:52 smithi026 ceph-mon[22362]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:53.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:52 smithi046 ceph-mon[23181]: pgmap v231: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:53.902 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:36:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:36:53.507Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:36:53.902 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:36:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:36:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:36:53.902 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:36:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:36:53.509Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:36:54.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:53 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:54.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:53 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:54.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:53 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:36:55.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:54 smithi026 ceph-mon[22362]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:55.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:54 smithi026 ceph-mon[27123]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:54 smithi046 ceph-mon[23181]: pgmap v232: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:36:55.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:36:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:36:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:36:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:56 smithi026 ceph-mon[27123]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:57.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:56 smithi026 ceph-mon[22362]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:57.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:56 smithi046 ceph-mon[23181]: pgmap v233: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:59.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:36:58 smithi026 ceph-mon[27123]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:59.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:36:58 smithi026 ceph-mon[22362]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:36:59.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:36:58 smithi046 ceph-mon[23181]: pgmap v234: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:01.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:00 smithi026 ceph-mon[27123]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:01.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:00 smithi026 ceph-mon[22362]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:01.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:00 smithi046 ceph-mon[23181]: pgmap v235: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:02.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:37:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:02] "GET /metrics HTTP/1.1" 200 207857 "" "Prometheus/2.33.4" 2024-04-30T09:37:03.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:02 smithi026 ceph-mon[22362]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:03.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:02 smithi026 ceph-mon[27123]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:02 smithi046 ceph-mon[23181]: pgmap v236: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:03.918 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:03 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:37:03.508Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:03.918 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:03 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:37:03.918 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:03 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:03.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:04.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:03 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:03 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:03 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:05.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:04 smithi026 ceph-mon[22362]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:05.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:04 smithi026 ceph-mon[27123]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:04 smithi046 ceph-mon[23181]: pgmap v237: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:05.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[24371]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:37:07.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:06 smithi026 ceph-mon[27123]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:07.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:06 smithi026 ceph-mon[22362]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:07.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:06 smithi046 ceph-mon[23181]: pgmap v238: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:09.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:08 smithi026 ceph-mon[27123]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:09.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:08 smithi026 ceph-mon[22362]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:09.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:08 smithi046 ceph-mon[23181]: pgmap v239: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:11.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:10 smithi026 ceph-mon[27123]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:11.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:10 smithi026 ceph-mon[22362]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:11.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:10 smithi046 ceph-mon[23181]: pgmap v240: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:11.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:11 smithi046 systemd[1]: Stopping Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:37:11.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:11 smithi046 bash[43336]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-30T09:37:12.090 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:11 smithi046 podman[43350]: 2024-04-30 09:37:11.810157178 +0000 UTC m=+0.161659862 container died 2b8d2d89eb233f694b8cbd22c8e1a1e3be911145c0a81ff44a3ddcbb24d47a4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, build-date=2022-05-03T08:36:31.336870, io.openshift.tags=base centos centos-stream, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., ceph=True, io.buildah.version=1.19.8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, GIT_CLEAN=True, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, architecture=x86_64) 2024-04-30T09:37:12.358 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 podman[43350]: 2024-04-30 09:37:12.15573576 +0000 UTC m=+0.507238441 container cleanup 2b8d2d89eb233f694b8cbd22c8e1a1e3be911145c0a81ff44a3ddcbb24d47a4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, GIT_CLEAN=True, RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, release=754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, vcs-type=git, GIT_BRANCH=HEAD, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.19.8, vendor=Red Hat, Inc.) 2024-04-30T09:37:12.358 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 bash[43350]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x 2024-04-30T09:37:12.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:37:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:12] "GET /metrics HTTP/1.1" 200 207857 "" "Prometheus/2.33.4" 2024-04-30T09:37:12.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 podman[43364]: 2024-04-30 09:37:12.35759053 +0000 UTC m=+0.542682401 container remove 2b8d2d89eb233f694b8cbd22c8e1a1e3be911145c0a81ff44a3ddcbb24d47a4e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , name=centos-stream, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, vendor=Red Hat, Inc., version=8, architecture=x86_64, release=754, GIT_CLEAN=True, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-30T09:37:12.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 bash[43377]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr.x" found: no such container 2024-04-30T09:37:12.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-30T09:37:12.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-30T09:37:12.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 systemd[1]: Stopped Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:37:12.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.x.service: Consumed 18.386s CPU time. 2024-04-30T09:37:12.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 systemd[1]: Starting Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:37:13.057 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:12 smithi046 ceph-mon[23181]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:13.057 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 podman[43478]: 2024-04-30 09:37:12.783982375 +0000 UTC m=+0.107374525 container create 53731c91d116a3f117c52a188ae444f1a71305b4efbc10948221f29ceab62a7d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T09:37:13.058 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:12 smithi046 podman[43478]: 2024-04-30 09:37:12.696236204 +0000 UTC m=+0.019628355 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:37:13.058 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:13 smithi046 podman[43478]: 2024-04-30 09:37:13.028174598 +0000 UTC m=+0.351566747 container init 53731c91d116a3f117c52a188ae444f1a71305b4efbc10948221f29ceab62a7d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-30T09:37:13.058 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:13 smithi046 podman[43478]: 2024-04-30 09:37:13.03099633 +0000 UTC m=+0.354388478 container start 53731c91d116a3f117c52a188ae444f1a71305b4efbc10948221f29ceab62a7d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=) 2024-04-30T09:37:13.058 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:13 smithi046 ceph-mgr[43498]: -- 172.21.15.46:0/1209361395 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55ff2755e5a0 con 0x55ff27533000 2024-04-30T09:37:13.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:12 smithi026 ceph-mon[27123]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:12 smithi026 ceph-mon[22362]: pgmap v241: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:13.392 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:13 smithi046 bash[43478]: 53731c91d116a3f117c52a188ae444f1a71305b4efbc10948221f29ceab62a7d 2024-04-30T09:37:13.392 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:13 smithi046 systemd[1]: Started Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:37:13.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:37:13.509Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=3 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:13.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:13.510Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T09:37:13.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:13.511Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:37:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:14 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:37:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:14 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:14 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:14 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:37:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:14 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:37:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:14 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:37:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:15 smithi046 ceph-mon[23181]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:15.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:15 smithi026 ceph-mon[27123]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:15.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:15 smithi026 ceph-mon[22362]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:17 smithi046 ceph-mon[23181]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:17.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:17 smithi026 ceph-mon[27123]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:17.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:17 smithi026 ceph-mon[22362]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:18 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:18 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:18.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:18 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:18.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:18 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:18.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:18 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:18.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:18 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:37:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:19 smithi046 ceph-mon[23181]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:19.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:19 smithi026 ceph-mon[27123]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:19.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:19 smithi026 ceph-mon[22362]: pgmap v244: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:20.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:37:20] ENGINE Bus STARTING 2024-04-30T09:37:20.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: CherryPy Checker: 2024-04-30T09:37:20.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: The Application mounted at '' has an empty config. 2024-04-30T09:37:20.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:37:20.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:37:20] ENGINE Serving on http://:::9283 2024-04-30T09:37:20.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:37:20] ENGINE Bus STARTED 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[27123]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[27123]: Standby manager daemon x restarted 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[27123]: Standby manager daemon x started 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[22362]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[22362]: Standby manager daemon x restarted 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[22362]: Standby manager daemon x started 2024-04-30T09:37:21.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:37:21.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:37:21.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:37:21.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:21 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:37:21.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:21 smithi046 ceph-mon[23181]: pgmap v245: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:21.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:21 smithi046 ceph-mon[23181]: Standby manager daemon x restarted 2024-04-30T09:37:21.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:21 smithi046 ceph-mon[23181]: Standby manager daemon x started 2024-04-30T09:37:21.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:21 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:37:21.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:21 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:37:21.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:21 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:37:21.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:21 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:37:22.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:22 smithi026 ceph-mon[22362]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-30T09:37:22.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:22 smithi026 ceph-mon[27123]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-30T09:37:22.574 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:37:22 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:22] "GET /metrics HTTP/1.1" 200 207857 "" "Prometheus/2.33.4" 2024-04-30T09:37:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:22 smithi046 ceph-mon[23181]: mgrmap e21: y(active, since 6m), standbys: x 2024-04-30T09:37:23.511 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:23 smithi026 ceph-mon[22362]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:23.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:23 smithi026 ceph-mon[27123]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:23 smithi046 ceph-mon[23181]: pgmap v246: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:23.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:37:23.510Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T09:37:23.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:23.512Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:23.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:23.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:37:24.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:24 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:24 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:24 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:25 smithi046 ceph-mon[23181]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:37:25.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:25 smithi026 ceph-mon[22362]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:25.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:25 smithi026 ceph-mon[27123]: pgmap v247: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:26.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:26 smithi046 ceph-mon[23181]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:27.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:26 smithi026 ceph-mon[22362]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:26 smithi026 ceph-mon[27123]: pgmap v248: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:27.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:37:27.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:37:27.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:37:27.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:37:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:37:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:37:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:37:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:37:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:37:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:37:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:37:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:37:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:28 smithi046 ceph-mon[23181]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:28 smithi026 ceph-mon[27123]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:28 smithi026 ceph-mon[22362]: pgmap v249: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:30.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:30 smithi046 ceph-mon[23181]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:31.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:30 smithi026 ceph-mon[27123]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:31.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:30 smithi026 ceph-mon[22362]: pgmap v250: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:32.577 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:37:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:32] "GET /metrics HTTP/1.1" 200 207858 "" "Prometheus/2.33.4" 2024-04-30T09:37:32.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:32 smithi026 ceph-mon[22362]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:32.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:32 smithi026 ceph-mon[27123]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:32 smithi046 ceph-mon[23181]: pgmap v251: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:33.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:33 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:33.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:33 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:33.849 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:37:33.511Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:33.849 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:33.849 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:33.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:37:33.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:33 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:34.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:34 smithi046 ceph-mon[23181]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:35.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:34 smithi026 ceph-mon[27123]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:35.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:34 smithi026 ceph-mon[22362]: pgmap v252: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:37:36.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:36 smithi046 ceph-mon[23181]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:36 smithi026 ceph-mon[22362]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:36 smithi026 ceph-mon[27123]: pgmap v253: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:38 smithi046 ceph-mon[23181]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:39.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:38 smithi026 ceph-mon[27123]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:39.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:38 smithi026 ceph-mon[22362]: pgmap v254: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:40.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:40 smithi046 ceph-mon[23181]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:40 smithi026 ceph-mon[27123]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:41.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:40 smithi026 ceph-mon[22362]: pgmap v255: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:42.586 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:37:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:42] "GET /metrics HTTP/1.1" 200 207858 "" "Prometheus/2.33.4" 2024-04-30T09:37:42.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:42 smithi026 ceph-mon[27123]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:42.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:42 smithi026 ceph-mon[22362]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:42.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:42 smithi046 ceph-mon[23181]: pgmap v256: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:43.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:43 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:43.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:43 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:43.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:43 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:37:43.512Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:43.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:43 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:43.513Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:43.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:43 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:43.514Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:37:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:43 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:45.076 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:44 smithi046 ceph-mon[23181]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:45.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:44 smithi026 ceph-mon[27123]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:45.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:44 smithi026 ceph-mon[22362]: pgmap v257: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:45.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:37:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:46 smithi026 ceph-mon[27123]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:46 smithi026 ceph-mon[22362]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:46 smithi046 ceph-mon[23181]: pgmap v258: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:49.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:48 smithi026 ceph-mon[27123]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:48 smithi026 ceph-mon[22362]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:49.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:48 smithi046 ceph-mon[23181]: pgmap v259: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:51.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:50 smithi026 ceph-mon[22362]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:51.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:50 smithi026 ceph-mon[27123]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:50 smithi046 ceph-mon[23181]: pgmap v260: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:52.595 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:37:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:52] "GET /metrics HTTP/1.1" 200 207862 "" "Prometheus/2.33.4" 2024-04-30T09:37:52.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:52 smithi026 ceph-mon[27123]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:52.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:52 smithi026 ceph-mon[22362]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:53.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:52 smithi046 ceph-mon[23181]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:53.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:53 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:53.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:53 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:53.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:37:53.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:53.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:53.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:37:53.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:37:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:37:53.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:37:54.059 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:53 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:37:55.076 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:54 smithi046 ceph-mon[23181]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:54 smithi026 ceph-mon[27123]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:55.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:54 smithi026 ceph-mon[22362]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:37:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:37:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:37:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:37:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:56 smithi026 ceph-mon[27123]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:56 smithi026 ceph-mon[22362]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:56 smithi046 ceph-mon[23181]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:37:58 smithi026 ceph-mon[27123]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:59.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:37:58 smithi026 ceph-mon[22362]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:37:59.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:37:58 smithi046 ceph-mon[23181]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:01.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:00 smithi026 ceph-mon[27123]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:01.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:00 smithi026 ceph-mon[22362]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:00 smithi046 ceph-mon[23181]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:02.598 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:38:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:02] "GET /metrics HTTP/1.1" 200 207862 "" "Prometheus/2.33.4" 2024-04-30T09:38:02.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:02 smithi026 ceph-mon[27123]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:02.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:02 smithi026 ceph-mon[22362]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:02 smithi046 ceph-mon[23181]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:03.771 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:03 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:38:03.513Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:38:03.771 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:03 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:03.515Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:38:03.771 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:03 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:03.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:03 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:03 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:04.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:03 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:05.076 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:04 smithi046 ceph-mon[23181]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:05.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:04 smithi026 ceph-mon[27123]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:04 smithi026 ceph-mon[22362]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:38:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:38:07.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:06 smithi026 ceph-mon[27123]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:07.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:06 smithi026 ceph-mon[22362]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:06 smithi046 ceph-mon[23181]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:09.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:08 smithi026 ceph-mon[27123]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:09.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:08 smithi026 ceph-mon[22362]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:09.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:08 smithi046 ceph-mon[23181]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:11.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:10 smithi026 ceph-mon[22362]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:11.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:10 smithi026 ceph-mon[27123]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:11.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:10 smithi046 ceph-mon[23181]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:12.608 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:38:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:12] "GET /metrics HTTP/1.1" 200 207862 "" "Prometheus/2.33.4" 2024-04-30T09:38:12.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:12 smithi026 ceph-mon[27123]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:12.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:12 smithi026 ceph-mon[22362]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:13.036 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:12 smithi046 ceph-mon[23181]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:13.785 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:38:13.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:13.785 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:13.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:38:13.785 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:13.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:14.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:13 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:14.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:13 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:13 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:15.076 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:14 smithi046 ceph-mon[23181]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:15.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:14 smithi026 ceph-mon[22362]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:15.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:14 smithi026 ceph-mon[27123]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:38:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:38:17.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:16 smithi026 ceph-mon[27123]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:17.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:16 smithi026 ceph-mon[22362]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:16 smithi046 ceph-mon[23181]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:17.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:17 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:38:17.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:17 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:38:17.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:17 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:38:17.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:17 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:38:17.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:17 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:38:17.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:17 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:38:17.884 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:17 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:38:17.884 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:17 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:38:17.884 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:17 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:38:19.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:18 smithi026 ceph-mon[27123]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:19.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:18 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:38:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:18 smithi026 ceph-mon[22362]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:18 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:38:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:18 smithi046 ceph-mon[23181]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:18 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' 2024-04-30T09:38:21.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:20 smithi026 ceph-mon[27123]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:21.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:20 smithi026 ceph-mon[22362]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:20 smithi046 ceph-mon[23181]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:22.615 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:38:22 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:22] "GET /metrics HTTP/1.1" 200 207858 "" "Prometheus/2.33.4" 2024-04-30T09:38:23.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:22 smithi026 ceph-mon[22362]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:23.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:22 smithi026 ceph-mon[27123]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:22 smithi046 ceph-mon[23181]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:23.897 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:38:23.514Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:23.897 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:23.516Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:23.897 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:23.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:38:24.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:23 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:24.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:23 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:23 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:25.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:24 smithi026 ceph-mon[27123]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:25.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:24 smithi026 ceph-mon[22362]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:25.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:24 smithi046 ceph-mon[23181]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:25.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:38:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:38:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:26 smithi026 ceph-mon[27123]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:26 smithi026 ceph-mon[22362]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:26 smithi046 ceph-mon[23181]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:28.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:38:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:27 smithi026 ceph-mon[22362]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:38:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:38:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:27 smithi026 ceph-mon[22362]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:38:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:38:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:27 smithi026 ceph-mon[27123]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:38:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:38:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:27 smithi026 ceph-mon[27123]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:38:28.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:38:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:27 smithi046 ceph-mon[23181]: from='mgr.24361 172.21.15.26:0/2346971668' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:38:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:38:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:27 smithi046 ceph-mon[23181]: from='mgr.24361 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:38:29.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:28 smithi026 ceph-mon[22362]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:29.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:28 smithi026 ceph-mon[27123]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:28 smithi046 ceph-mon[23181]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:31.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:30 smithi026 ceph-mon[22362]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:31.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:30 smithi026 ceph-mon[27123]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:30 smithi046 ceph-mon[23181]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:32.621 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:38:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:32] "GET /metrics HTTP/1.1" 200 207856 "" "Prometheus/2.33.4" 2024-04-30T09:38:33.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:32 smithi026 ceph-mon[22362]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:32 smithi026 ceph-mon[27123]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:32 smithi046 ceph-mon[23181]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:33.919 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:38:33.515Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:33.919 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:33.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:33.919 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:33.517Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:38:34.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:33 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:33 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:33 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:34 smithi026 ceph-mon[27123]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:34 smithi026 ceph-mon[22362]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:34 smithi046 ceph-mon[23181]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:35.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:38:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:38:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:36 smithi026 ceph-mon[27123]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:36 smithi026 ceph-mon[22362]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:36 smithi046 ceph-mon[23181]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:39.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:38 smithi026 ceph-mon[27123]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:39.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:38 smithi026 ceph-mon[22362]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:39.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:38 smithi046 ceph-mon[23181]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:40 smithi026 ceph-mon[27123]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:40 smithi026 ceph-mon[22362]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:41.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:40 smithi046 ceph-mon[23181]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:42.630 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:38:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:42] "GET /metrics HTTP/1.1" 200 207856 "" "Prometheus/2.33.4" 2024-04-30T09:38:43.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:42 smithi026 ceph-mon[27123]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:42 smithi026 ceph-mon[22362]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:42 smithi046 ceph-mon[23181]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:43.939 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:43 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:38:43.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:38:43.939 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:43 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:38:43.940 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:43 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:43.518Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:44.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:43 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:44.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:43 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:44.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:43 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:45.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:44 smithi026 ceph-mon[27123]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:45.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:44 smithi026 ceph-mon[22362]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:45.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:44 smithi046 ceph-mon[23181]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:45.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:38:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:45] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:38:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:46 smithi026 ceph-mon[27123]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:46 smithi026 ceph-mon[22362]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:46 smithi046 ceph-mon[23181]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:49.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:48 smithi046 ceph-mon[23181]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:48 smithi026 ceph-mon[27123]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:49.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:48 smithi026 ceph-mon[22362]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:50 smithi046 ceph-mon[23181]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:51.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:50 smithi026 ceph-mon[27123]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:50 smithi026 ceph-mon[22362]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:52.637 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:38:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:52] "GET /metrics HTTP/1.1" 200 207857 "" "Prometheus/2.33.4" 2024-04-30T09:38:53.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:52 smithi046 ceph-mon[23181]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:53.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:52 smithi026 ceph-mon[22362]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:52 smithi026 ceph-mon[27123]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:53.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:38:53.516Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:53.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:53.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:38:53.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:38:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:38:53.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:38:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:53 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:53 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:53 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:38:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:54 smithi046 ceph-mon[23181]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:55.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:38:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:38:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:38:55.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:54 smithi026 ceph-mon[22362]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:55.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:54 smithi026 ceph-mon[27123]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:38:56.387 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T09:38:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:56 smithi046 ceph-mon[23181]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:57.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:56 smithi026 ceph-mon[27123]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:57.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:56 smithi026 ceph-mon[22362]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:58.288 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:38:58.288 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (6m) 2m ago 7m 23.9M - ba2b418f427c 7b0e3dcf393a 2024-04-30T09:38:58.288 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (7m) 101s ago 7m 58.9M - 8.3.5 dad864ee21e9 9ecc79f0568a 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 running (6m) 2m ago 6m 51.3M - 3.5 e1d6a67b021e 68f3038ef8d6 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283 running (105s) 101s ago 12m 42.3M - 19.0.0-3357-g3a873b0c c50cf5fa633a 53731c91d116 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (13m) 2m ago 13m 460M - 17.2.0 e1d6a67b021e 780f8f988439 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (13m) 2m ago 13m 52.6M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (12m) 101s ago 12m 39.5M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (12m) 2m ago 12m 43.7M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (8m) 2m ago 8m 16.1M - 1dbe0e931976 d712368b56f8 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (8m) 101s ago 8m 19.6M - 1dbe0e931976 727cc4da58b3 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (11m) 2m ago 11m 49.8M 4017M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (11m) 2m ago 11m 49.1M 4017M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (10m) 2m ago 10m 47.9M 4017M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (10m) 2m ago 10m 47.8M 4017M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (10m) 101s ago 10m 51.6M 4273M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (9m) 101s ago 9m 48.0M 4273M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (9m) 101s ago 9m 46.9M 4273M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:38:58.289 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (8m) 101s ago 8m 49.5M 4273M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:38:58.290 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (6m) 101s ago 8m 64.7M - 514e6a882f6e f54f9679bceb 2024-04-30T09:38:58.290 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (6m) 2m ago 6m 70.2M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:38:58.290 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (6m) 101s ago 6m 70.3M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:38:59.012 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions' 2024-04-30T09:38:59.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:58 smithi046 ceph-mon[23181]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:59.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:58 smithi026 ceph-mon[27123]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:38:59.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:58 smithi026 ceph-mon[22362]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:00.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:38:59 smithi046 ceph-mon[23181]: from='client.24934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:39:00.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:38:59 smithi026 ceph-mon[27123]: from='client.24934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:39:00.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:38:59 smithi026 ceph-mon[22362]: from='client.24934 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:39:01.066 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:39:01.066 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 1 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:39:01.067 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:39:01.068 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:39:01.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:39:01.068 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:39:01.068 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 14, 2024-04-30T09:39:01.068 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 1 2024-04-30T09:39:01.068 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:39:01.068 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:39:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:00 smithi046 ceph-mon[23181]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:01.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:00 smithi026 ceph-mon[27123]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:01.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:00 smithi026 ceph-mon[22362]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:01.697 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph -s' 2024-04-30T09:39:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:01 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/752427712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:39:02.272 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:01 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/752427712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:39:02.272 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:01 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/752427712' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:39:02.637 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:39:02] "GET /metrics HTTP/1.1" 200 207858 "" "Prometheus/2.33.4" 2024-04-30T09:39:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:02 smithi046 ceph-mon[23181]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:03.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:02 smithi026 ceph-mon[27123]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:03.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:02 smithi026 ceph-mon[22362]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:03.794 INFO:teuthology.orchestra.run.smithi026.stdout: cluster: 2024-04-30T09:39:03.794 INFO:teuthology.orchestra.run.smithi026.stdout: id: 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:39:03.794 INFO:teuthology.orchestra.run.smithi026.stdout: health: HEALTH_OK 2024-04-30T09:39:03.794 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:39:03.794 INFO:teuthology.orchestra.run.smithi026.stdout: services: 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: mon: 3 daemons, quorum a,c,b (age 12m) 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: mgr: y(active, since 8m), standbys: x 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: osd: 8 osds: 8 up (since 8m), 8 in (since 9m) 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: data: 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: pools: 6 pools, 161 pgs 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: objects: 209 objects, 585 KiB 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: usage: 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: pgs: 161 active+clean 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:39:03.795 INFO:teuthology.orchestra.run.smithi026.stdout: io: 2024-04-30T09:39:03.796 INFO:teuthology.orchestra.run.smithi026.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-30T09:39:03.796 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:39:03.796 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:03 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:39:03.517Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:39:03.796 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:03 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:39:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:39:03.796 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:03 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:39:03.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:39:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:03 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:39:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:03 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2205004511' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:39:04.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:03 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:39:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:03 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2205004511' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:39:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:03 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:39:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:03 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2205004511' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:39:04.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T09:39:05.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:04 smithi026 ceph-mon[22362]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:05.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:04 smithi026 ceph-mon[27123]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:04 smithi046 ceph-mon[23181]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:05.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:39:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:39:06.726 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-30T09:39:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:06 smithi046 ceph-mon[23181]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:06 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1511356286' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:39:07.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:06 smithi026 ceph-mon[27123]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:07.315 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:06 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1511356286' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:39:07.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:06 smithi026 ceph-mon[22362]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:07.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:06 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1511356286' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:39:07.340 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 2'"'"'' 2024-04-30T09:39:09.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:08 smithi026 ceph-mon[27123]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:09.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:08 smithi026 ceph-mon[22362]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:09.462 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:39:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:08 smithi046 ceph-mon[23181]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:10.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:09 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/393123026' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:39:10.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:09 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/393123026' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:39:10.218 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph mgr fail' 2024-04-30T09:39:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:09 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/393123026' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:39:11.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:10 smithi026 ceph-mon[27123]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:11.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:10 smithi026 ceph-mon[22362]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:10 smithi046 ceph-mon[23181]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:12.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:12 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:39:12] "GET /metrics HTTP/1.1" 200 207858 "" "Prometheus/2.33.4" 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-mon[23181]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3990778040' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-mon[23181]: osdmap e96: 8 total, 8 up, 8 in 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:13] ENGINE Bus STOPPING 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:13] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:13] ENGINE Bus STOPPED 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:13] ENGINE Bus STARTING 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:13] ENGINE Serving on http://:::9283 2024-04-30T09:39:13.286 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:13] ENGINE Bus STARTED 2024-04-30T09:39:13.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-mon[22362]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:13.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3990778040' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:39:13.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:39:13.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-mon[22362]: osdmap e96: 8 total, 8 up, 8 in 2024-04-30T09:39:13.327 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:13.002+0000 7f7cc7adb700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-30T09:39:13.327 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ignoring --setuser ceph since I am not root 2024-04-30T09:39:13.327 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ignoring --setgroup ceph since I am not root 2024-04-30T09:39:13.327 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-mgr[22595]: -- 172.21.15.26:0/3048021315 <== mon.1 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5591173f6340 con 0x5591173cf000 2024-04-30T09:39:13.327 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:13.190+0000 7f9595add000 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-30T09:39:13.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-mon[27123]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:13.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3990778040' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:39:13.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:39:13.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-mon[27123]: osdmap e96: 8 total, 8 up, 8 in 2024-04-30T09:39:13.644 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:13.331+0000 7f9595add000 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:39:13.644 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:13.433+0000 7f9595add000 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-30T09:39:13.644 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:39:13.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:39:13.644 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:39:13.519Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T09:39:13.644 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:39:13.522Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:39:13.672 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'sleep 180' 2024-04-30T09:39:13.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:13.769+0000 7f9595add000 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-30T09:39:13.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:13 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:13.858+0000 7f9595add000 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-30T09:39:14.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-30T09:39:14.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: mgrmap e22: x(active, starting, since 0.625469s) 2024-04-30T09:39:14.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:39:14.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:39:14.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:39:14.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:39:14.282 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: Manager daemon x is now available 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: Queued rgw.foo for migration 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: mgrmap e22: x(active, starting, since 0.625469s) 2024-04-30T09:39:14.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: Manager daemon x is now available 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: Queued rgw.foo for migration 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:14 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-30T09:39:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: mgrmap e22: x(active, starting, since 0.625469s) 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:39:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: Manager daemon x is now available 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: Queued rgw.foo for migration 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: No Migration is needed for rgw spec: {'placement': {'count': 2}, 'service_id': 'foo', 'service_name': 'rgw.foo', 'service_type': 'rgw', 'spec': {'rgw_frontend_port': 8000, 'rgw_realm': 'r', 'rgw_zone': 'z'}} 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:14.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:14 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-mon[27123]: Deploying cephadm binary to smithi046 2024-04-30T09:39:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-mon[27123]: Deploying cephadm binary to smithi026 2024-04-30T09:39:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-mon[27123]: mgrmap e23: x(active, since 1.63479s) 2024-04-30T09:39:15.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-mon[27123]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:15.354 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:15.086+0000 7f9595add000 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-30T09:39:15.355 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:15.265+0000 7f9595add000 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-30T09:39:15.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-mon[22362]: Deploying cephadm binary to smithi046 2024-04-30T09:39:15.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-mon[22362]: Deploying cephadm binary to smithi026 2024-04-30T09:39:15.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-mon[22362]: mgrmap e23: x(active, since 1.63479s) 2024-04-30T09:39:15.355 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-mon[22362]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:39:15] "GET /metrics HTTP/1.1" 200 34888 "" "Prometheus/2.33.4" 2024-04-30T09:39:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:15 smithi046 ceph-mon[23181]: Deploying cephadm binary to smithi046 2024-04-30T09:39:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:15 smithi046 ceph-mon[23181]: Deploying cephadm binary to smithi026 2024-04-30T09:39:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:15 smithi046 ceph-mon[23181]: mgrmap e23: x(active, since 1.63479s) 2024-04-30T09:39:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:15 smithi046 ceph-mon[23181]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:15.685 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:15.353+0000 7f9595add000 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-30T09:39:15.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:15.684+0000 7f9595add000 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-30T09:39:15.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:15.780+0000 7f9595add000 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-30T09:39:15.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:15 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:15.877+0000 7f9595add000 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-30T09:39:16.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:16.131+0000 7f9595add000 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[27123]: [30/Apr/2024:09:39:14] ENGINE Bus STARTING 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[27123]: [30/Apr/2024:09:39:14] ENGINE Serving on https://172.21.15.46:7150 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[27123]: [30/Apr/2024:09:39:14] ENGINE Serving on http://172.21.15.46:8765 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[27123]: [30/Apr/2024:09:39:14] ENGINE Bus STARTED 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[27123]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[27123]: mgrmap e24: x(active, since 2s) 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[22362]: [30/Apr/2024:09:39:14] ENGINE Bus STARTING 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[22362]: [30/Apr/2024:09:39:14] ENGINE Serving on https://172.21.15.46:7150 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[22362]: [30/Apr/2024:09:39:14] ENGINE Serving on http://172.21.15.46:8765 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[22362]: [30/Apr/2024:09:39:14] ENGINE Bus STARTED 2024-04-30T09:39:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[22362]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-mon[22362]: mgrmap e24: x(active, since 2s) 2024-04-30T09:39:16.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:16 smithi046 ceph-mon[23181]: [30/Apr/2024:09:39:14] ENGINE Bus STARTING 2024-04-30T09:39:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:16 smithi046 ceph-mon[23181]: [30/Apr/2024:09:39:14] ENGINE Serving on https://172.21.15.46:7150 2024-04-30T09:39:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:16 smithi046 ceph-mon[23181]: [30/Apr/2024:09:39:14] ENGINE Serving on http://172.21.15.46:8765 2024-04-30T09:39:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:16 smithi046 ceph-mon[23181]: [30/Apr/2024:09:39:14] ENGINE Bus STARTED 2024-04-30T09:39:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:16 smithi046 ceph-mon[23181]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:16 smithi046 ceph-mon[23181]: mgrmap e24: x(active, since 2s) 2024-04-30T09:39:16.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:16.600+0000 7f9595add000 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-30T09:39:17.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:17 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:17.115+0000 7f9595add000 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-30T09:39:17.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:17 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:17.214+0000 7f9595add000 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-30T09:39:17.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:17 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:17.526+0000 7f9595add000 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-30T09:39:17.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:17 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:17.621+0000 7f9595add000 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-30T09:39:18.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:18 smithi026 ceph-mon[22362]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:18.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:18 smithi026 ceph-mon[22362]: mgrmap e25: x(active, since 4s) 2024-04-30T09:39:18.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:18 smithi026 ceph-mon[27123]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:18.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:18 smithi026 ceph-mon[27123]: mgrmap e25: x(active, since 4s) 2024-04-30T09:39:18.289 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:18 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:18.048+0000 7f9595add000 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-30T09:39:18.289 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:18 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:18.189+0000 7f9595add000 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-30T09:39:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:18 smithi046 ceph-mon[23181]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:18 smithi046 ceph-mon[23181]: mgrmap e25: x(active, since 4s) 2024-04-30T09:39:18.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:18 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:18.287+0000 7f9595add000 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-30T09:39:18.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:18 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:18.409+0000 7f9595add000 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-30T09:39:18.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:18 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:18.503+0000 7f9595add000 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-30T09:39:18.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked down and back up at {{ $value | humanize }} times once a minute for 5 minutes. This could indicate a network issue (latency, packet drop, disruption) on the clusters \"cluster network\". Check the network environment on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSD's to flap (mark each other out)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.26:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:39:18.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on(job) group_left() avg by(job) (ceph_osd_numpg\n > 0)) / on(job) group_left() avg by(job) (ceph_osd_numpg > 0)) * on(ceph_daemon)\n group_left(hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: |\n OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by more than 30% from average PG count.\n summary: PG allocations are not balanced across devices\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.26:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:39:19.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:19 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:19.118+0000 7f9595add000 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-30T09:39:19.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:19 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:19.506+0000 7f9595add000 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-30T09:39:19.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:19 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:39:19] ENGINE Bus STARTING 2024-04-30T09:39:19.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:19 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: CherryPy Checker: 2024-04-30T09:39:19.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:19 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: The Application mounted at '' has an empty config. 2024-04-30T09:39:19.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:19 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: 2024-04-30T09:39:19.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:19 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:39:19] ENGINE Serving on http://:::9283 2024-04-30T09:39:19.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:19 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: [30/Apr/2024:09:39:19] ENGINE Bus STARTED 2024-04-30T09:39:20.383 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:20 smithi046 ceph-mon[23181]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:20.383 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:20 smithi046 ceph-mon[23181]: Standby manager daemon y started 2024-04-30T09:39:20.383 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:20 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-30T09:39:20.383 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:20 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:39:20.383 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:20 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-30T09:39:20.383 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:20 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:39:20.392 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[27123]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:20.392 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[27123]: Standby manager daemon y started 2024-04-30T09:39:20.392 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-30T09:39:20.392 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:39:20.392 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-30T09:39:20.392 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:39:20.392 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:39:20.114Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=6 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:39:20.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[22362]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:20.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[22362]: Standby manager daemon y started 2024-04-30T09:39:20.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-30T09:39:20.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:39:20.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-30T09:39:20.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:20 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/1574237126' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:39:21.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:21 smithi026 ceph-mon[27123]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-30T09:39:21.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:21 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:39:21.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:21 smithi026 ceph-mon[22362]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-30T09:39:21.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:21 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:39:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:21 smithi046 ceph-mon[23181]: mgrmap e26: x(active, since 7s), standbys: y 2024-04-30T09:39:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:21 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:39:22.324 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:22 smithi026 ceph-mon[27123]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:39:22.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:22 smithi026 ceph-mon[22362]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:39:22.443 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:22 smithi046 ceph-mon[23181]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:39:22.655 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:22 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:39:22] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:39:23.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[27123]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.520 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-mon[22362]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:39:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:23 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:23 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:23 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:39:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:23 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:23 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:23 smithi046 ceph-mon[23181]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:39:23.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=error ts=2024-04-30T09:39:23.518Z caller=dispatch.go:354 component=dispatcher msg="Notify for alerts failed" num_alerts=6 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:39:23.949 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:39:23.520Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.46:8443/api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.46 because it doesn't contain any IP SANs" 2024-04-30T09:39:23.949 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=warn ts=2024-04-30T09:39:23.521Z caller=notify.go:724 component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://172.21.15.26:8443//api/prometheus_receiver\": x509: cannot validate certificate for 172.21.15.26 because it doesn't contain any IP SANs" 2024-04-30T09:39:24.644 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.644 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.644 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.644 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.644 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.644 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.644 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.644 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.644 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi026 to 2993M 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:24.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:24 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi026 to 2993M 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:24.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi026 to 2993M 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:24.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:24 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:39:25.390 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:39:25] "GET /metrics HTTP/1.1" 200 34888 "" "Prometheus/2.33.4" 2024-04-30T09:39:25.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: Adjusting osd_memory_target on smithi046 to 3249M 2024-04-30T09:39:25.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: Adjusting osd_memory_target on smithi046 to 3249M 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:39:25.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:39:25.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:25 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: Adjusting osd_memory_target on smithi046 to 3249M 2024-04-30T09:39:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:39:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:39:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:39:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:39:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:25.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:25 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:26.176 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:25 smithi026 systemd[1]: Stopping Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:26.176 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:25 smithi026 bash[56238]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-30T09:39:26.176 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:25 smithi026 podman[56251]: 2024-04-30 09:39:25.926422435 +0000 UTC m=+0.135390846 container died d712368b56f8c644f5dd6568f9f6d5ecf956bb9cde0e8f5441d121c07d46a890 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:26.446 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 podman[56251]: 2024-04-30 09:39:26.274150625 +0000 UTC m=+0.483119034 container cleanup d712368b56f8c644f5dd6568f9f6d5ecf956bb9cde0e8f5441d121c07d46a890 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:26.446 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 bash[56251]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a 2024-04-30T09:39:26.446 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 bash[56278]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter.a" found: no such container 2024-04-30T09:39:26.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:26 smithi026 ceph-mon[27123]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-30T09:39:26.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:26 smithi026 ceph-mon[27123]: Reconfiguring daemon node-exporter.a on smithi026 2024-04-30T09:39:26.818 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 podman[56265]: 2024-04-30 09:39:26.550279112 +0000 UTC m=+0.627858767 container remove d712368b56f8c644f5dd6568f9f6d5ecf956bb9cde0e8f5441d121c07d46a890 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:26.818 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-30T09:39:26.818 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-30T09:39:26.819 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 systemd[1]: Stopped Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:26.819 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 4.040s CPU time. 2024-04-30T09:39:26.819 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 systemd[1]: Starting Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:26.819 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 podman[56343]: 2024-04-30 09:39:26.69169179 +0000 UTC m=+0.018287351 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-30T09:39:26.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:26 smithi026 ceph-mon[22362]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-30T09:39:26.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:26 smithi026 ceph-mon[22362]: Reconfiguring daemon node-exporter.a on smithi026 2024-04-30T09:39:26.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:26 smithi046 ceph-mon[23181]: Reconfiguring node-exporter.a (dependencies changed)... 2024-04-30T09:39:26.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:26 smithi046 ceph-mon[23181]: Reconfiguring daemon node-exporter.a on smithi026 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:26 smithi026 podman[56343]: 2024-04-30 09:39:26.817998023 +0000 UTC m=+0.144593598 container create b6af5622fe4c99e5282e593ca75798b6601decd7a73deca8c29d8b11d9c82066 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 podman[56343]: 2024-04-30 09:39:27.036868158 +0000 UTC m=+0.363463696 container init b6af5622fe4c99e5282e593ca75798b6601decd7a73deca8c29d8b11d9c82066 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 podman[56343]: 2024-04-30 09:39:27.03965458 +0000 UTC m=+0.366250114 container start b6af5622fe4c99e5282e593ca75798b6601decd7a73deca8c29d8b11d9c82066 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.043Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.043Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.043Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.043Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=arp 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-30T09:39:27.121 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-30T09:39:27.122 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-30T09:39:27.122 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-30T09:39:27.122 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-30T09:39:27.122 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-30T09:39:27.122 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=edac 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-30T09:39:27.123 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-30T09:39:27.124 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-30T09:39:27.124 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-30T09:39:27.124 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-30T09:39:27.124 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-30T09:39:27.124 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-30T09:39:27.124 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-30T09:39:27.124 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=os 2024-04-30T09:39:27.124 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-30T09:39:27.124 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.044Z caller=node_exporter.go:115 level=info collector=stat 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:115 level=info collector=time 2024-04-30T09:39:27.125 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-30T09:39:27.126 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:115 level=info collector=uname 2024-04-30T09:39:27.126 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-30T09:39:27.126 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-30T09:39:27.126 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-30T09:39:27.126 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-30T09:39:27.126 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[56357]: ts=2024-04-30T09:39:27.045Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-30T09:39:27.126 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 bash[56343]: b6af5622fe4c99e5282e593ca75798b6601decd7a73deca8c29d8b11d9c82066 2024-04-30T09:39:27.446 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:39:27 smithi026 systemd[1]: Started Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:27.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:27 smithi046 ceph-mon[23181]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:39:27.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:27 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:27.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:27 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:28.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-mon[22362]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:39:28.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:28.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:28.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-mon[27123]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:39:28.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:28.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:27 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:28.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:28 smithi026 ceph-mon[22362]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-30T09:39:28.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:28 smithi026 ceph-mon[22362]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-30T09:39:28.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:28 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:28.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:28 smithi026 ceph-mon[27123]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-30T09:39:28.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:28 smithi026 ceph-mon[27123]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-30T09:39:28.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:28 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:28 smithi046 ceph-mon[23181]: Reconfiguring alertmanager.a (dependencies changed)... 2024-04-30T09:39:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:28 smithi046 ceph-mon[23181]: Reconfiguring daemon alertmanager.a on smithi026 2024-04-30T09:39:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:28 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:30.162 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:29 smithi026 ceph-mon[27123]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:39:30.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:29 smithi026 ceph-mon[22362]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:39:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:29 smithi046 ceph-mon[23181]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:39:30.696 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:30 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:30.696 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:30 smithi026 bash[56531]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-30T09:39:31.017 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[47070]: level=info ts=2024-04-30T09:39:30.757Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-30T09:39:31.017 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:30 smithi026 podman[56544]: 2024-04-30 09:39:30.778040823 +0000 UTC m=+0.130484489 container died 7b0e3dcf393a466e6330dab6626416cb1ae0851c5a31b0c925efb048397db8cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:31.291 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 podman[56544]: 2024-04-30 09:39:31.107097869 +0000 UTC m=+0.459541549 container cleanup 7b0e3dcf393a466e6330dab6626416cb1ae0851c5a31b0c925efb048397db8cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:31.292 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 bash[56544]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a 2024-04-30T09:39:31.693 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 podman[56558]: 2024-04-30 09:39:31.291500882 +0000 UTC m=+0.508892151 container remove 7b0e3dcf393a466e6330dab6626416cb1ae0851c5a31b0c925efb048397db8cb (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:31.693 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 podman[56558]: 2024-04-30 09:39:31.383886545 +0000 UTC m=+0.601277813 volume remove c48ee1fc834c96cd1b6660f45e55129d2c5a6376b48b859a5939df5e0db3d8e2 2024-04-30T09:39:31.693 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 bash[56570]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager.a" found: no such container 2024-04-30T09:39:31.693 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-30T09:39:31.694 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:31.694 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.074s CPU time. 2024-04-30T09:39:31.694 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 systemd[1]: Starting Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:31.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:31 smithi026 ceph-mon[27123]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:39:31.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:31 smithi026 ceph-mon[22362]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:39:31.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 podman[56638]: 2024-04-30 09:39:31.693169216 +0000 UTC m=+0.094909140 volume create 000f990602969dedfb5f80dd884ae561dc0b1a910c03a83889d086ad45a2c1ab 2024-04-30T09:39:31.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 podman[56638]: 2024-04-30 09:39:31.617548458 +0000 UTC m=+0.019288397 image pull ba2b418f427c0636d654de8757e830c80168e76482bcc46bb2138e569d6c91d4 quay.io/prometheus/alertmanager:v0.23.0 2024-04-30T09:39:31.947 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:31 smithi026 podman[56638]: 2024-04-30 09:39:31.793868487 +0000 UTC m=+0.195608452 container create 1648d8f22b65737e1674c98f8752ca1d04dd0dcaaa8155d726c633474c9ba0ef (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:31 smithi046 ceph-mon[23181]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:39:32.199 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 podman[56638]: 2024-04-30 09:39:32.080281898 +0000 UTC m=+0.482021824 container init 1648d8f22b65737e1674c98f8752ca1d04dd0dcaaa8155d726c633474c9ba0ef (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:32.199 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 podman[56638]: 2024-04-30 09:39:32.083341156 +0000 UTC m=+0.485081080 container start 1648d8f22b65737e1674c98f8752ca1d04dd0dcaaa8155d726c633474c9ba0ef (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:32.199 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:32.094Z caller=main.go:225 msg="Starting Alertmanager" version="(version=0.23.0, branch=HEAD, revision=61046b17771a57cfd4c4a51be370ab930a4d7d54)" 2024-04-30T09:39:32.199 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:32.094Z caller=main.go:226 build_context="(go=go1.16.7, user=root@e21a959be8d2, date=20210825-10:48:55)" 2024-04-30T09:39:32.199 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:32.096Z caller=cluster.go:184 component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-04-30T09:39:32.199 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:32.097Z caller=cluster.go:671 component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-30T09:39:32.199 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:32.123Z caller=coordinator.go:113 component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-30T09:39:32.200 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:32.123Z caller=coordinator.go:126 component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-30T09:39:32.200 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:32.126Z caller=main.go:518 msg=Listening address=:9093 2024-04-30T09:39:32.200 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:32.126Z caller=tls_config.go:191 msg="TLS is disabled." http2=false 2024-04-30T09:39:32.200 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 bash[56638]: 1648d8f22b65737e1674c98f8752ca1d04dd0dcaaa8155d726c633474c9ba0ef 2024-04-30T09:39:32.455 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:39:32 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[22591]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:39:32] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:39:32.455 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:32 smithi026 systemd[1]: Started Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:32.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:32.315Z caller=manager.go:609 level=warn component="rule manager" group=pools msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on(pool_id) group_right()\n ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: |\n Pool '{{ $labels.name }}' will be full in less than 5 days assuming the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed it's capacity\n" err="found duplicate series for the match group {pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"172.21.15.46:9283\", job=\"ceph\", pool_id=\"1\"}, {instance=\"172.21.15.26:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:39:33.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:33.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:33.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[27123]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[27123]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[27123]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[27123]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[22362]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:39:33.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:33.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:33.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:33.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[22362]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:39:33.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[22362]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:39:33.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:33 smithi026 ceph-mon[22362]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:33 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:33 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:33 smithi046 ceph-mon[23181]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:39:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:33 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:33 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:33 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:33 smithi046 ceph-mon[23181]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:39:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:33 smithi046 ceph-mon[23181]: from='client.14841 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:39:33.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:33 smithi046 ceph-mon[23181]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:34.428 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:34.097Z caller=cluster.go:696 component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000550018s 2024-04-30T09:39:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:39:35] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-04-30T09:39:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:39:35.288+0000 7f78edcbc700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:39:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:39:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:39:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:39:35.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:39:35.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:39:35.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:39:35.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return future.result(timeout) 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.__get_result() 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: raise self._exception 2024-04-30T09:39:35.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: use_current_daemon_image=reconfig, 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:39:35.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:39:35.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:39:35.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:39:35.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:39:35.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:39:35.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:39:35.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:35.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:36.438 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:36 smithi046 ceph-mon[23181]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:36.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:36 smithi026 ceph-mon[22362]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:36.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:36 smithi026 ceph-mon[27123]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:39:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:37 smithi046 ceph-mon[23181]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:39:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:39:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:39:37.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return future.result(timeout) 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.__get_result() 2024-04-30T09:39:37.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: raise self._exception 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:39:37.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:37 smithi046 ceph-mon[23181]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:39:37.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:37 smithi046 ceph-mon[23181]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:37 smithi026 ceph-mon[22362]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:39:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:39:37.448 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:39:37.449 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:39:37.450 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:37 smithi026 ceph-mon[22362]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:37 smithi026 ceph-mon[22362]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:37 smithi026 ceph-mon[27123]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:39:37.451 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:39:37.452 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:39:37.453 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:39:37.454 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:37 smithi026 ceph-mon[27123]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:39:37.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:37 smithi026 ceph-mon[27123]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:39:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-mon[23181]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:38.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:38 smithi026 ceph-mon[27123]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:38 smithi026 ceph-mon[22362]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:38.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 systemd[1]: Stopping Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:38.877 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 bash[46447]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:39:39.194 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-30T09:39:39.194 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-30T09:39:39.194 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-30T09:39:39.194 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-30T09:39:39.194 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-30T09:39:39.194 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-30T09:39:39.194 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-30T09:39:39.195 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-30T09:39:39.195 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-30T09:39:39.195 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-30T09:39:39.195 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-30T09:39:39.195 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[40117]: ts=2024-04-30T09:39:38.876Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-30T09:39:39.195 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:38 smithi046 podman[46460]: 2024-04-30 09:39:38.886230651 +0000 UTC m=+0.117193373 container died f54f9679bceb4d6ac5a87db25cd211c06787f9a1161db5300859e5a28ec29d71 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:39.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 podman[46460]: 2024-04-30 09:39:39.292523035 +0000 UTC m=+0.523485755 container cleanup f54f9679bceb4d6ac5a87db25cd211c06787f9a1161db5300859e5a28ec29d71 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:39.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 bash[46460]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a 2024-04-30T09:39:39.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 bash[46488]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:39:39.770 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 podman[46474]: 2024-04-30 09:39:39.518726217 +0000 UTC m=+0.634473679 container remove f54f9679bceb4d6ac5a87db25cd211c06787f9a1161db5300859e5a28ec29d71 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:39.770 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-30T09:39:39.770 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 systemd[1]: Stopped Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:39.770 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 2.020s CPU time. 2024-04-30T09:39:39.770 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 systemd[1]: Starting Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:39.770 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 podman[46559]: 2024-04-30 09:39:39.659208029 +0000 UTC m=+0.019950063 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-30T09:39:39.770 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:39 smithi046 podman[46559]: 2024-04-30 09:39:39.769854461 +0000 UTC m=+0.130596492 container create 1b17cf12c30b4c9b4966720562e86995797e813d3eef733867f1fade97491095 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:40.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:40 smithi026 ceph-mon[22362]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:40.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:40 smithi026 ceph-mon[27123]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-mon[23181]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:40.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 podman[46559]: 2024-04-30 09:39:40.02234278 +0000 UTC m=+0.383084814 container init 1b17cf12c30b4c9b4966720562e86995797e813d3eef733867f1fade97491095 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:40.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 podman[46559]: 2024-04-30 09:39:40.025146274 +0000 UTC m=+0.385888314 container start 1b17cf12c30b4c9b4966720562e86995797e813d3eef733867f1fade97491095 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:39:40.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.049Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-30T09:39:40.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.049Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-30T09:39:40.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.049Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-30T09:39:40.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.049Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi046 (none))" 2024-04-30T09:39:40.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.049Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-30T09:39:40.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.049Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-30T09:39:40.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.050Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-30T09:39:40.485 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.051Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-30T09:39:40.485 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.051Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-30T09:39:40.485 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.054Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-30T09:39:40.485 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.054Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.809µs 2024-04-30T09:39:40.485 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.054Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-30T09:39:40.485 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 bash[46559]: 1b17cf12c30b4c9b4966720562e86995797e813d3eef733867f1fade97491095 2024-04-30T09:39:40.485 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 systemd[1]: Started Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:40.485 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.269Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-30T09:39:40.886 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:40 smithi046 systemd[1]: Stopping Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:40.887 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:40 smithi046 bash[46682]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-30T09:39:40.887 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.538Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-30T09:39:40.887 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.538Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-30T09:39:40.887 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.538Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.123µs wal_replay_duration=483.856266ms total_replay_duration=483.902006ms 2024-04-30T09:39:40.887 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.541Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-30T09:39:40.888 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.541Z caller=main.go:947 level=info msg="TSDB started" 2024-04-30T09:39:40.888 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.541Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-30T09:39:40.888 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.566Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.596058ms db_storage=646ns remote_storage=1.159µs web_handler=324ns query_engine=669ns scrape=377.182µs scrape_sd=113.848µs notify=10.397µs notify_sd=12.045µs rules=23.618572ms 2024-04-30T09:39:40.888 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:40.566Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-30T09:39:41.193 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:40 smithi046 podman[46695]: 2024-04-30 09:39:40.886029528 +0000 UTC m=+0.110783183 container died 727cc4da58b386ae792dd1338a37247198bf495d31bb36dae6fb422d58093ae5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:39:41.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:41.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:41.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-mon[23181]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-30T09:39:41.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-mon[23181]: Reconfiguring daemon node-exporter.b on smithi046 2024-04-30T09:39:41.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-mon[23181]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:41.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 podman[46695]: 2024-04-30 09:39:41.292528737 +0000 UTC m=+0.517282391 container cleanup 727cc4da58b386ae792dd1338a37247198bf495d31bb36dae6fb422d58093ae5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:39:41.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 bash[46695]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b 2024-04-30T09:39:41.485 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 bash[46722]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter.b" found: no such container 2024-04-30T09:39:41.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:41.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:41.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[27123]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-30T09:39:41.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[27123]: Reconfiguring daemon node-exporter.b on smithi046 2024-04-30T09:39:41.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[27123]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:41.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:41.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:41.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[22362]: Reconfiguring node-exporter.b (dependencies changed)... 2024-04-30T09:39:41.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[22362]: Reconfiguring daemon node-exporter.b on smithi046 2024-04-30T09:39:41.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:41 smithi026 ceph-mon[22362]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:41.754 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 podman[46709]: 2024-04-30 09:39:41.527455902 +0000 UTC m=+0.641657451 container remove 727cc4da58b386ae792dd1338a37247198bf495d31bb36dae6fb422d58093ae5 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:39:41.754 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-30T09:39:41.754 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-30T09:39:41.754 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 systemd[1]: Stopped Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:41.754 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 4.068s CPU time. 2024-04-30T09:39:41.755 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 systemd[1]: Starting Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:42.006 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 podman[46799]: 2024-04-30 09:39:41.753571269 +0000 UTC m=+0.109291871 container create 23b6576e6bb541b3f77460e4ef986aa6c2771b3d737e75768dd95a18f2b9197c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:39:42.006 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 podman[46799]: 2024-04-30 09:39:41.662778415 +0000 UTC m=+0.018499019 image pull 1dbe0e931976487e20e5cfb272087e08a9779c88fd5e9617ed7042dd9751ec26 quay.io/prometheus/node-exporter:v1.3.1 2024-04-30T09:39:42.006 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 podman[46799]: 2024-04-30 09:39:41.939241589 +0000 UTC m=+0.294962191 container init 23b6576e6bb541b3f77460e4ef986aa6c2771b3d737e75768dd95a18f2b9197c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 podman[46799]: 2024-04-30 09:39:41.942128141 +0000 UTC m=+0.297848746 container start 23b6576e6bb541b3f77460e4ef986aa6c2771b3d737e75768dd95a18f2b9197c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.945Z caller=node_exporter.go:182 level=info msg="Starting node_exporter" version="(version=1.3.1, branch=HEAD, revision=a2321e7b940ddcff26873612bccdf7cd4c42b6b6)" 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.945Z caller=node_exporter.go:183 level=info msg="Build context" build_context="(go=go1.17.3, user=root@243aafa5525c, date=20211205-11:09:49)" 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.945Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+)($|/) 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.945Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:108 level=info msg="Enabled collectors" 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=arp 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=bcache 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=bonding 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=btrfs 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=conntrack 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=cpu 2024-04-30T09:39:42.007 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=cpufreq 2024-04-30T09:39:42.008 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=diskstats 2024-04-30T09:39:42.008 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=dmi 2024-04-30T09:39:42.008 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=edac 2024-04-30T09:39:42.008 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=entropy 2024-04-30T09:39:42.008 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=fibrechannel 2024-04-30T09:39:42.008 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=filefd 2024-04-30T09:39:42.008 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=filesystem 2024-04-30T09:39:42.008 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=hwmon 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=infiniband 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=ipvs 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=loadavg 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=mdadm 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=meminfo 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=netclass 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=netdev 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=netstat 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=nfs 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=nfsd 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=nvme 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=os 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=powersupplyclass 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=pressure 2024-04-30T09:39:42.009 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=rapl 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=schedstat 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=sockstat 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=softnet 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=stat 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=tapestats 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=textfile 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=thermal_zone 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=time 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=udp_queues 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=uname 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.946Z caller=node_exporter.go:115 level=info collector=vmstat 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.947Z caller=node_exporter.go:115 level=info collector=xfs 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.947Z caller=node_exporter.go:115 level=info collector=zfs 2024-04-30T09:39:42.010 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.947Z caller=node_exporter.go:199 level=info msg="Listening on" address=:9100 2024-04-30T09:39:42.011 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:41 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[46815]: ts=2024-04-30T09:39:41.947Z caller=tls_config.go:195 level=info msg="TLS is disabled." http2=false 2024-04-30T09:39:42.011 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:42 smithi046 bash[46799]: 23b6576e6bb541b3f77460e4ef986aa6c2771b3d737e75768dd95a18f2b9197c 2024-04-30T09:39:42.307 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:39:42 smithi046 systemd[1]: Started Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:42.446 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:39:42 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:39:42.099Z caller=cluster.go:688 component=cluster msg="gossip settled; proceeding" elapsed=10.002525122s 2024-04-30T09:39:43.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[22362]: Reconfiguring grafana.a (dependencies changed)... 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[27123]: Reconfiguring grafana.a (dependencies changed)... 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:43 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:39:43.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:43 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:43 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:43 smithi046 ceph-mon[23181]: Reconfiguring grafana.a (dependencies changed)... 2024-04-30T09:39:43.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:43 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:43 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:43.324 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:43 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:39:44.397 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:44 smithi046 ceph-mon[23181]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-30T09:39:44.397 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:44 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:39:44.397 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:44 smithi046 ceph-mon[23181]: Reconfiguring daemon grafana.a on smithi046 2024-04-30T09:39:44.397 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:44 smithi046 ceph-mon[23181]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:39:44.398 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:44 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[22362]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[22362]: Reconfiguring daemon grafana.a on smithi046 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[22362]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[27123]: Regenerating cephadm self-signed grafana TLS certificates 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[27123]: Reconfiguring daemon grafana.a on smithi046 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[27123]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:39:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:44 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:45.984 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:45 smithi046 systemd[1]: Stopping Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:45.984 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:45 smithi046 bash[46989]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-30T09:39:45.984 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[39324]: t=2024-04-30T09:39:45+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-30T09:39:45.984 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:45 smithi046 podman[47002]: 2024-04-30 09:39:45.766453712 +0000 UTC m=+0.130580855 container died 9ecc79f0568a8eca6b6764be7c932b8acf85eb35345df72ca1428ba1029ab7f8 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.expose-services=, description=Ceph Grafana Container, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-03-28T10:36:18.413762, io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.component=ubi8-container, io.buildah.version=1.24.2, version=8.5, architecture=x86_64, name=ubi8) 2024-04-30T09:39:46.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:46 smithi026 ceph-mon[27123]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:39:46.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:46 smithi026 ceph-mon[22362]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:39:46.334 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-mon[23181]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:39:46.334 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 podman[47002]: 2024-04-30 09:39:46.157560258 +0000 UTC m=+0.521687399 container cleanup 9ecc79f0568a8eca6b6764be7c932b8acf85eb35345df72ca1428ba1029ab7f8 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , io.buildah.version=1.24.2, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, com.redhat.component=ubi8-container, io.openshift.expose-services=, build-date=2022-03-28T10:36:18.413762, version=8.5, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.openshift.tags=base rhel8, vcs-type=git, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-04-30T09:39:46.335 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 bash[47002]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a 2024-04-30T09:39:46.602 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 podman[47016]: 2024-04-30 09:39:46.333944658 +0000 UTC m=+0.563122921 container remove 9ecc79f0568a8eca6b6764be7c932b8acf85eb35345df72ca1428ba1029ab7f8 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, io.buildah.version=1.24.2, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=236.1648460182, build-date=2022-03-28T10:36:18.413762, description=Ceph Grafana Container, vcs-type=git, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, architecture=x86_64, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-04-30T09:39:46.602 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 bash[47029]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana.a" found: no such container 2024-04-30T09:39:46.602 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-30T09:39:46.602 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 systemd[1]: Stopped Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:46.602 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@grafana.a.service: Consumed 3.414s CPU time. 2024-04-30T09:39:46.602 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 systemd[1]: Starting Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:39:46.602 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 podman[47097]: 2024-04-30 09:39:46.601401455 +0000 UTC m=+0.103919005 container create 9bd49a22598807b17d78024541c8dff4cc28e0aa7fc9ee5e88424f66630178b5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, vcs-type=git, io.buildah.version=1.24.2, release=236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, distribution-scope=public, name=ubi8, version=8.5, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc.) 2024-04-30T09:39:46.870 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 podman[47097]: 2024-04-30 09:39:46.516444644 +0000 UTC m=+0.018962156 image pull dad864ee21e98e69f4029d1e417aa085001566be0d322fbc75bc6f29b0050c01 quay.io/ceph/ceph-grafana:8.3.5 2024-04-30T09:39:46.871 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 podman[47097]: 2024-04-30 09:39:46.838430058 +0000 UTC m=+0.340947582 container init 9bd49a22598807b17d78024541c8dff4cc28e0aa7fc9ee5e88424f66630178b5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, distribution-scope=public, description=Ceph Grafana Container, io.buildah.version=1.24.2, vcs-type=git, architecture=x86_64, release=236.1648460182, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, io.openshift.expose-services=, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc.) 2024-04-30T09:39:46.871 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 podman[47097]: 2024-04-30 09:39:46.841676681 +0000 UTC m=+0.344194199 container start 9bd49a22598807b17d78024541c8dff4cc28e0aa7fc9ee5e88424f66630178b5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, maintainer=Paul Cuzner , version=8.5, architecture=x86_64, io.buildah.version=1.24.2, release=236.1648460182, com.redhat.component=ubi8-container, distribution-scope=public, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-03-28T10:36:18.413762, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, vcs-type=git, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182) 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:46] ENGINE Bus STOPPING 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STOPPED 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STARTING 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Serving on http://:::9283 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STARTED 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STOPPING 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STOPPED 2024-04-30T09:39:47.217 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STARTING 2024-04-30T09:39:47.218 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Serving on http://:::9283 2024-04-30T09:39:47.218 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STARTED 2024-04-30T09:39:47.218 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STOPPING 2024-04-30T09:39:47.218 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="The state of unified alerting is still not defined. The decision will be made during as we run the database migrations" logger=settings 2024-04-30T09:39:47.218 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=warn msg="falling back to legacy setting of 'min_interval_seconds'; please use the configuration option in the `unified_alerting` section if Grafana 8 alerts are enabled." logger=settings 2024-04-30T09:39:47.218 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Config loaded from" logger=settings file=/usr/share/grafana/conf/defaults.ini 2024-04-30T09:39:47.218 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Config loaded from" logger=settings file=/etc/grafana/grafana.ini 2024-04-30T09:39:47.218 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Config overridden from Environment variable" logger=settings var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Path Home" logger=settings path=/usr/share/grafana 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Path Data" logger=settings path=/var/lib/grafana 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Path Logs" logger=settings path=/var/log/grafana 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Path Plugins" logger=settings path=/var/lib/grafana/plugins 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Path Provisioning" logger=settings path=/etc/grafana/provisioning 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="App mode production" logger=settings 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Connecting to DB" logger=sqlstore dbtype=sqlite3 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=warn msg="SQLite database file has broader permissions than it should" logger=sqlstore path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Starting DB migrations" logger=migrator 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="migrations completed" logger=migrator performed=0 skipped=377 duration=524.855µs 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Created default organization" logger=sqlstore 2024-04-30T09:39:47.219 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Initialising plugins" logger=plugin.manager 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=input 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 bash[47097]: 9bd49a22598807b17d78024541c8dff4cc28e0aa7fc9ee5e88424f66630178b5 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 systemd[1]: Started Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=grafana-piechart-panel 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Plugin registered" logger=plugin.manager pluginId=vonage-status-panel 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:46+0000 lvl=info msg="Live Push Gateway initialization" logger=live.push_http 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:47+0000 lvl=info msg="deleted datasource based on configuration" logger=provisioning.datasources name=Dashboard1 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:47+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:47+0000 lvl=info msg="inserting datasource from configuration " logger=provisioning.datasources name=Loki uid=P8E80F9AEF21F6940 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:47+0000 lvl=info msg="warming cache for startup" logger=ngalert 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:47+0000 lvl=info msg="HTTP Server Listen" logger=http.server address=[::]:3000 protocol=https subUrl= socket= 2024-04-30T09:39:47.220 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:39:47+0000 lvl=info msg="starting MultiOrg Alertmanager" logger=ngalert.multiorg.alertmanager 2024-04-30T09:39:47.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:39:47.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STOPPED 2024-04-30T09:39:47.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STARTING 2024-04-30T09:39:47.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Serving on http://:::9283 2024-04-30T09:39:47.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:39:47] ENGINE Bus STARTED 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:39:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046:3000"}]: dispatch 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046:3000"}]: dispatch 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3304898439' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4063723858' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1375005854"}]: dispatch 2024-04-30T09:39:48.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046:3000"}]: dispatch 2024-04-30T09:39:48.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046:3000"}]: dispatch 2024-04-30T09:39:48.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:48.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:39:48.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3304898439' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:39:48.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:47 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4063723858' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1375005854"}]: dispatch 2024-04-30T09:39:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:39:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:39:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-30T09:39:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2024-04-30T09:39:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046:3000"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046:3000"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3304898439' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:39:48.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:47 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4063723858' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1375005854"}]: dispatch 2024-04-30T09:39:49.414 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:49 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4063723858' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1375005854"}]': finished 2024-04-30T09:39:49.414 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:49 smithi046 ceph-mon[23181]: osdmap e97: 8 total, 8 up, 8 in 2024-04-30T09:39:49.414 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:49 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1018700068' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1501562524"}]: dispatch 2024-04-30T09:39:49.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:49 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4063723858' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1375005854"}]': finished 2024-04-30T09:39:49.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:49 smithi026 ceph-mon[27123]: osdmap e97: 8 total, 8 up, 8 in 2024-04-30T09:39:49.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:49 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1018700068' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1501562524"}]: dispatch 2024-04-30T09:39:49.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:49 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4063723858' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1375005854"}]': finished 2024-04-30T09:39:49.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:49 smithi026 ceph-mon[22362]: osdmap e97: 8 total, 8 up, 8 in 2024-04-30T09:39:49.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:49 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1018700068' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1501562524"}]: dispatch 2024-04-30T09:39:50.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:50 smithi026 ceph-mon[27123]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:50.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:50 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1018700068' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1501562524"}]': finished 2024-04-30T09:39:50.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:50 smithi026 ceph-mon[27123]: osdmap e98: 8 total, 8 up, 8 in 2024-04-30T09:39:50.279 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:50 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1154983881' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2789192078"}]: dispatch 2024-04-30T09:39:50.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:50 smithi026 ceph-mon[22362]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:50.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:50 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1018700068' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1501562524"}]': finished 2024-04-30T09:39:50.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:50 smithi026 ceph-mon[22362]: osdmap e98: 8 total, 8 up, 8 in 2024-04-30T09:39:50.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:50 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1154983881' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2789192078"}]: dispatch 2024-04-30T09:39:50.335 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:50 smithi046 ceph-mon[23181]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:39:50.335 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:50 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1018700068' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1501562524"}]': finished 2024-04-30T09:39:50.335 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:50 smithi046 ceph-mon[23181]: osdmap e98: 8 total, 8 up, 8 in 2024-04-30T09:39:50.335 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:50 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1154983881' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2789192078"}]: dispatch 2024-04-30T09:39:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:51 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1154983881' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2789192078"}]': finished 2024-04-30T09:39:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:51 smithi046 ceph-mon[23181]: osdmap e99: 8 total, 8 up, 8 in 2024-04-30T09:39:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:51 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3495500085' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2789192078"}]: dispatch 2024-04-30T09:39:51.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:51 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2789192078"}]: dispatch 2024-04-30T09:39:51.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:51 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:51.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:51 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:51.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1154983881' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2789192078"}]': finished 2024-04-30T09:39:51.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[27123]: osdmap e99: 8 total, 8 up, 8 in 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3495500085' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2789192078"}]: dispatch 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2789192078"}]: dispatch 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1154983881' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2789192078"}]': finished 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[22362]: osdmap e99: 8 total, 8 up, 8 in 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3495500085' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2789192078"}]: dispatch 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2789192078"}]: dispatch 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:51 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-30T09:39:52.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2789192078"}]': finished 2024-04-30T09:39:52.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: osdmap e100: 8 total, 8 up, 8 in 2024-04-30T09:39:52.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/579900362' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/449056828"}]: dispatch 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/449056828"}]: dispatch 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2789192078"}]': finished 2024-04-30T09:39:52.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: osdmap e100: 8 total, 8 up, 8 in 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/579900362' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/449056828"}]: dispatch 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/449056828"}]: dispatch 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:52.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:52 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2789192078"}]': finished 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: osdmap e100: 8 total, 8 up, 8 in 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/579900362' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/449056828"}]: dispatch 2024-04-30T09:39:52.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/449056828"}]: dispatch 2024-04-30T09:39:52.383 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:52.383 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:39:52.383 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:52 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:39:53.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[27123]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:39:53.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[27123]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:39:53.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/449056828"}]': finished 2024-04-30T09:39:53.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[27123]: osdmap e101: 8 total, 8 up, 8 in 2024-04-30T09:39:53.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/878200874' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/347975072"}]: dispatch 2024-04-30T09:39:53.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[22362]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:39:53.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[22362]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:39:53.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/449056828"}]': finished 2024-04-30T09:39:53.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[22362]: osdmap e101: 8 total, 8 up, 8 in 2024-04-30T09:39:53.326 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:53 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/878200874' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/347975072"}]: dispatch 2024-04-30T09:39:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:53 smithi046 ceph-mon[23181]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:39:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:53 smithi046 ceph-mon[23181]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:39:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:53 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/449056828"}]': finished 2024-04-30T09:39:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:53 smithi046 ceph-mon[23181]: osdmap e101: 8 total, 8 up, 8 in 2024-04-30T09:39:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:53 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/878200874' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/347975072"}]: dispatch 2024-04-30T09:39:53.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:39:54.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:54 smithi026 ceph-mon[27123]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:39:54.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:54 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/878200874' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/347975072"}]': finished 2024-04-30T09:39:54.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:54 smithi026 ceph-mon[27123]: osdmap e102: 8 total, 8 up, 8 in 2024-04-30T09:39:54.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:54 smithi026 ceph-mon[22362]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:39:54.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:54 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/878200874' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/347975072"}]': finished 2024-04-30T09:39:54.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:54 smithi026 ceph-mon[22362]: osdmap e102: 8 total, 8 up, 8 in 2024-04-30T09:39:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:54 smithi046 ceph-mon[23181]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 64 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:39:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:54 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/878200874' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/347975072"}]': finished 2024-04-30T09:39:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:54 smithi046 ceph-mon[23181]: osdmap e102: 8 total, 8 up, 8 in 2024-04-30T09:39:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:39:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:39:55] "GET /metrics HTTP/1.1" 200 37880 "" "Prometheus/2.33.4" 2024-04-30T09:39:56.409 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:56 smithi046 ceph-mon[23181]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:39:56.409 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:56 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:56.410 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:56 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:56.410 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:56 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:39:56.410 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:56 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:39:56.410 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:56 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:39:56.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[27123]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:39:56.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:56.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:56.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:39:56.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:39:56.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:39:56.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[22362]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:39:56.442 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:56.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:39:56.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:39:56.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:39:56.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:56 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:39:57.347 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:57 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:39:57.347 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:57 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:39:57.347 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:57 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1209433496' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:39:57.347 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:57 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3657849759' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2762166568"}]: dispatch 2024-04-30T09:39:57.347 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:57 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2762166568"}]: dispatch 2024-04-30T09:39:57.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:39:57.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:39:57.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1209433496' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:39:57.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3657849759' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2762166568"}]: dispatch 2024-04-30T09:39:57.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2762166568"}]: dispatch 2024-04-30T09:39:57.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:39:57.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:39:57.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1209433496' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:39:57.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3657849759' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2762166568"}]: dispatch 2024-04-30T09:39:57.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:57 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2762166568"}]: dispatch 2024-04-30T09:39:58.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:58 smithi026 ceph-mon[27123]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:58.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:58 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2762166568"}]': finished 2024-04-30T09:39:58.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:58 smithi026 ceph-mon[27123]: osdmap e103: 8 total, 8 up, 8 in 2024-04-30T09:39:58.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:58 smithi026 ceph-mon[22362]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:58.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:58 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2762166568"}]': finished 2024-04-30T09:39:58.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:58 smithi026 ceph-mon[22362]: osdmap e103: 8 total, 8 up, 8 in 2024-04-30T09:39:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:58 smithi046 ceph-mon[23181]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:39:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:58 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/2762166568"}]': finished 2024-04-30T09:39:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:58 smithi046 ceph-mon[23181]: osdmap e103: 8 total, 8 up, 8 in 2024-04-30T09:39:58.913 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:39:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:39:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:39:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:39:59 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:59.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:39:59 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:39:59.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:39:59 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:00.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[27123]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 770 B/s rd, 0 op/s 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[27123]: overall HEALTH_OK 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[22362]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 770 B/s rd, 0 op/s 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:40:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:40:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:00 smithi026 ceph-mon[22362]: overall HEALTH_OK 2024-04-30T09:40:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:00 smithi046 ceph-mon[23181]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 69 MiB used, 715 GiB / 715 GiB avail; 770 B/s rd, 0 op/s 2024-04-30T09:40:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:00 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:00 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:00 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:00 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:00 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:40:00.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:00 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:40:00.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:00 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:40:00.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:00 smithi046 ceph-mon[23181]: overall HEALTH_OK 2024-04-30T09:40:02.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:01 smithi026 ceph-mon[27123]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:40:02.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:01 smithi026 ceph-mon[22362]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:40:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:01 smithi046 ceph-mon[23181]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:40:03.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:03.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:04 smithi026 ceph-mon[27123]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 1 op/s 2024-04-30T09:40:04.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:04 smithi026 ceph-mon[22362]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 1 op/s 2024-04-30T09:40:04.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:04 smithi046 ceph-mon[23181]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 955 B/s rd, 1 op/s 2024-04-30T09:40:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:40:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:40:05] "GET /metrics HTTP/1.1" 200 37875 "" "Prometheus/2.33.4" 2024-04-30T09:40:06.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:06 smithi026 ceph-mon[22362]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-04-30T09:40:06.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:06 smithi026 ceph-mon[27123]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-04-30T09:40:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:06 smithi046 ceph-mon[23181]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-04-30T09:40:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:07 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:07 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:07 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:08.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:08 smithi026 ceph-mon[27123]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-04-30T09:40:08.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:08 smithi026 ceph-mon[22362]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-04-30T09:40:08.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:08 smithi046 ceph-mon[23181]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 921 B/s rd, 0 op/s 2024-04-30T09:40:08.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:10.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:10 smithi026 ceph-mon[27123]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 769 B/s rd, 0 op/s 2024-04-30T09:40:10.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:10 smithi026 ceph-mon[22362]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 769 B/s rd, 0 op/s 2024-04-30T09:40:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:10 smithi046 ceph-mon[23181]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 769 B/s rd, 0 op/s 2024-04-30T09:40:12.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:11 smithi026 ceph-mon[27123]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:11 smithi026 ceph-mon[22362]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:11 smithi046 ceph-mon[23181]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:13.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:13.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:14 smithi026 ceph-mon[27123]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:14 smithi026 ceph-mon[22362]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:14 smithi046 ceph-mon[23181]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:15.415 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:40:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:40:15] "GET /metrics HTTP/1.1" 200 37880 "" "Prometheus/2.33.4" 2024-04-30T09:40:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:16 smithi026 ceph-mon[27123]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:16 smithi026 ceph-mon[22362]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:16 smithi046 ceph-mon[23181]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:17.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:17 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:17.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:17 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:17 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:18 smithi026 ceph-mon[27123]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:18 smithi026 ceph-mon[22362]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:18 smithi046 ceph-mon[23181]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:18.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:20.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:20 smithi026 ceph-mon[27123]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:20 smithi026 ceph-mon[22362]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:20 smithi046 ceph-mon[23181]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:22.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:21 smithi026 ceph-mon[27123]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:21 smithi026 ceph-mon[22362]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:21 smithi046 ceph-mon[23181]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:23.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:23.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:24.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:24 smithi026 ceph-mon[27123]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:24 smithi026 ceph-mon[22362]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:24 smithi046 ceph-mon[23181]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:40:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:40:25] "GET /metrics HTTP/1.1" 200 37880 "" "Prometheus/2.33.4" 2024-04-30T09:40:26.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:26 smithi026 ceph-mon[27123]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:26.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:26 smithi026 ceph-mon[22362]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:26 smithi046 ceph-mon[23181]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:27.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:27 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:27.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:27 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:27 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:28.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:28 smithi026 ceph-mon[27123]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:28.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:28 smithi026 ceph-mon[22362]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:28.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:28 smithi046 ceph-mon[23181]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:28.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:29.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:29 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:29.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:29 smithi026 ceph-mon[27123]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:29.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:29 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:29.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:29 smithi026 ceph-mon[22362]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:29 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:29 smithi046 ceph-mon[23181]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:31 smithi026 ceph-mon[27123]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:31 smithi026 ceph-mon[22362]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:31 smithi046 ceph-mon[23181]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:33.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:33.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:34.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:34 smithi026 ceph-mon[27123]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:34 smithi026 ceph-mon[22362]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:34 smithi046 ceph-mon[23181]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:40:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:40:35] "GET /metrics HTTP/1.1" 200 37880 "" "Prometheus/2.33.4" 2024-04-30T09:40:36.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:36 smithi026 ceph-mon[27123]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:36.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:36 smithi026 ceph-mon[22362]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:36.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:36 smithi046 ceph-mon[23181]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:37.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:37 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:37 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:37.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:37 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:38.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:38 smithi026 ceph-mon[27123]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:38 smithi026 ceph-mon[22362]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:38 smithi046 ceph-mon[23181]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:38.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:39.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:39 smithi026 ceph-mon[27123]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:39.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:39 smithi026 ceph-mon[22362]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:39 smithi046 ceph-mon[23181]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:41 smithi026 ceph-mon[27123]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:41 smithi026 ceph-mon[22362]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:41 smithi046 ceph-mon[23181]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:43.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:43 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:44.243 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:44 smithi046 ceph-mon[23181]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:44.243 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:44 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:44.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:44 smithi026 ceph-mon[27123]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:44 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:44 smithi026 ceph-mon[22362]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:44 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:45.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:40:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:40:45] "GET /metrics HTTP/1.1" 200 37879 "" "Prometheus/2.33.4" 2024-04-30T09:40:46.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:46 smithi026 ceph-mon[22362]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:46.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:46 smithi026 ceph-mon[27123]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:46 smithi046 ceph-mon[23181]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:47.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:47 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:47.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:47 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:47 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:48.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:48 smithi026 ceph-mon[27123]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:48.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:48 smithi026 ceph-mon[22362]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:48 smithi046 ceph-mon[23181]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:48.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:49 smithi026 ceph-mon[27123]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:49.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:49 smithi026 ceph-mon[22362]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:49 smithi046 ceph-mon[23181]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:52.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:51 smithi026 ceph-mon[27123]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:51 smithi026 ceph-mon[22362]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:51 smithi046 ceph-mon[23181]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:53.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:54.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:54 smithi026 ceph-mon[27123]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:54 smithi026 ceph-mon[22362]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:54 smithi046 ceph-mon[23181]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:40:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:40:55] "GET /metrics HTTP/1.1" 200 37879 "" "Prometheus/2.33.4" 2024-04-30T09:40:56.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:56 smithi026 ceph-mon[22362]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:56.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:56 smithi026 ceph-mon[27123]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:56.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:56 smithi046 ceph-mon[23181]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:57.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:57 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:57.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:57 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:57 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:40:58.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:58 smithi026 ceph-mon[22362]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:58.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:58 smithi026 ceph-mon[27123]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:58 smithi046 ceph-mon[23181]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:40:58.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:40:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:40:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:40:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:59 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:59.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:40:59 smithi026 ceph-mon[27123]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:59.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:59 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:59.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:40:59 smithi026 ceph-mon[22362]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:40:59.675 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:59 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:40:59.675 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:40:59 smithi046 ceph-mon[23181]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:00.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:00 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:41:00.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:00 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:41:00.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:00 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:41:01.955 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:01 smithi046 ceph-mon[23181]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:02.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:01 smithi026 ceph-mon[27123]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:02.015 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:01 smithi026 ceph-mon[22362]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:03.939 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:04.199 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:04 smithi046 ceph-mon[23181]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:04.199 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:04 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:04.199 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:04 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:04 smithi026 ceph-mon[27123]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:04 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:04.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:04 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:04 smithi026 ceph-mon[22362]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:04 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:04 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:05.467 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:41:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:41:05] "GET /metrics HTTP/1.1" 200 37880 "" "Prometheus/2.33.4" 2024-04-30T09:41:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:05 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:05 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:05 smithi046 ceph-mon[23181]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:05 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:41:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:05 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:41:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:05 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[27123]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[22362]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:05.796 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:41:05.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:41:05.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:05 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:41:07.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:06 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:07.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:06 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:06 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:08.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:07 smithi026 ceph-mon[27123]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:08.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:07 smithi026 ceph-mon[22362]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:08.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:07 smithi046 ceph-mon[23181]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:08.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:08.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:10.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:10 smithi026 ceph-mon[22362]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:10.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:10 smithi026 ceph-mon[27123]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:10 smithi046 ceph-mon[23181]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:12.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:11 smithi026 ceph-mon[22362]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:12.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:11 smithi026 ceph-mon[27123]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:11 smithi046 ceph-mon[23181]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:13.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:14 smithi026 ceph-mon[27123]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:14 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:14 smithi026 ceph-mon[22362]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:14 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:14 smithi046 ceph-mon[23181]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:14 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:41:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:41:15] "GET /metrics HTTP/1.1" 200 37877 "" "Prometheus/2.33.4" 2024-04-30T09:41:16.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:16 smithi026 ceph-mon[27123]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:16 smithi026 ceph-mon[22362]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:16 smithi046 ceph-mon[23181]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:17.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:17 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:17 smithi026 ceph-mon[27123]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:17 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:17 smithi026 ceph-mon[22362]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:17 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:17 smithi046 ceph-mon[23181]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:18.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:20.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:20 smithi026 ceph-mon[22362]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:20.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:20 smithi026 ceph-mon[27123]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:20 smithi046 ceph-mon[23181]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:22.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:21 smithi026 ceph-mon[27123]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:21 smithi026 ceph-mon[22362]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:21 smithi046 ceph-mon[23181]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:23.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:24.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:24 smithi026 ceph-mon[27123]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:24 smithi026 ceph-mon[22362]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:24 smithi046 ceph-mon[23181]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:25.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:25 smithi026 ceph-mon[27123]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:25.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:25 smithi026 ceph-mon[22362]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:25 smithi046 ceph-mon[23181]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:41:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:41:25] "GET /metrics HTTP/1.1" 200 37877 "" "Prometheus/2.33.4" 2024-04-30T09:41:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:26 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:26 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:26 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:27 smithi026 ceph-mon[27123]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:27 smithi026 ceph-mon[22362]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:27 smithi046 ceph-mon[23181]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:28 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:28.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:28 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:28 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:30.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:29 smithi026 ceph-mon[27123]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:29 smithi026 ceph-mon[22362]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:29 smithi046 ceph-mon[23181]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:32.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:31 smithi026 ceph-mon[27123]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:32.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:31 smithi026 ceph-mon[22362]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:31 smithi046 ceph-mon[23181]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:33.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:34.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:34 smithi026 ceph-mon[27123]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:34 smithi026 ceph-mon[22362]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:34 smithi046 ceph-mon[23181]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:35.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:35 smithi026 ceph-mon[27123]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:35.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:35 smithi026 ceph-mon[22362]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:35.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:35 smithi046 ceph-mon[23181]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:41:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:41:35] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-04-30T09:41:37.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:36 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:36 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:36 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:38.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:37 smithi026 ceph-mon[22362]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:38.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:37 smithi026 ceph-mon[27123]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:37 smithi046 ceph-mon[23181]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:38.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:39.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:39 smithi026 ceph-mon[22362]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:39.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:39 smithi026 ceph-mon[27123]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:39 smithi046 ceph-mon[23181]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:42.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:41 smithi026 ceph-mon[22362]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:42.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:41 smithi026 ceph-mon[27123]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:41 smithi046 ceph-mon[23181]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:43 smithi026 ceph-mon[22362]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:43 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:43.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:43 smithi026 ceph-mon[27123]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:43.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:43 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:43 smithi046 ceph-mon[23181]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:43 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:43.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:43 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:45.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:45 smithi026 ceph-mon[22362]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:45.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:45 smithi026 ceph-mon[27123]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:45.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:45 smithi046 ceph-mon[23181]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:45.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:41:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:41:45] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.33.4" 2024-04-30T09:41:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:46 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:46 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:46 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:48.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:47 smithi026 ceph-mon[27123]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:48.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:47 smithi026 ceph-mon[22362]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:48.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:47 smithi046 ceph-mon[23181]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:48.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:49.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:49 smithi026 ceph-mon[22362]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:49 smithi026 ceph-mon[27123]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:49 smithi046 ceph-mon[23181]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:52.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:51 smithi026 ceph-mon[27123]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:51 smithi026 ceph-mon[22362]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:52.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:51 smithi046 ceph-mon[23181]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:53.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:53 smithi026 ceph-mon[27123]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:53.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:53 smithi026 ceph-mon[22362]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:53 smithi046 ceph-mon[23181]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:53.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:55.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:55 smithi026 ceph-mon[22362]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:55.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:55 smithi026 ceph-mon[27123]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:55 smithi046 ceph-mon[23181]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:41:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:41:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:41:55] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.33.4" 2024-04-30T09:41:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:56 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:57.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:56 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:57.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:56 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:41:58.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:57 smithi026 ceph-mon[27123]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:58.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:57 smithi026 ceph-mon[22362]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:57 smithi046 ceph-mon[23181]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:41:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:58 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:58.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:41:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:41:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:41:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:58 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:41:59.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:58 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:00.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:41:59 smithi026 ceph-mon[27123]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:00.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:41:59 smithi026 ceph-mon[22362]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:41:59 smithi046 ceph-mon[23181]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:02.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:01 smithi026 ceph-mon[27123]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:02.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:01 smithi026 ceph-mon[22362]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:01 smithi046 ceph-mon[23181]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:03.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:03 smithi026 ceph-mon[27123]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:03.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:03 smithi026 ceph-mon[22362]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:03 smithi046 ceph-mon[23181]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:03.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:03.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:05.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:05 smithi026 ceph-mon[22362]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:05.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:05 smithi026 ceph-mon[27123]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:05 smithi046 ceph-mon[23181]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:42:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:42:05] "GET /metrics HTTP/1.1" 200 37882 "" "Prometheus/2.33.4" 2024-04-30T09:42:06.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:42:06.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:06.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:06.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:42:06.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:06.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:42:06.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:06.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:06.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:42:06.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:06 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:06 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:42:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:06 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:06 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:06 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:42:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:06 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:07.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:07 smithi026 ceph-mon[27123]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:07.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:07 smithi026 ceph-mon[22362]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:07 smithi046 ceph-mon[23181]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:08.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:09.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:09 smithi026 ceph-mon[27123]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:09.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:09 smithi026 ceph-mon[22362]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:09 smithi046 ceph-mon[23181]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:12.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:11 smithi026 ceph-mon[27123]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:11 smithi026 ceph-mon[22362]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:11 smithi046 ceph-mon[23181]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:13 smithi026 ceph-mon[27123]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:13 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:13 smithi026 ceph-mon[22362]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:13 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:13 smithi046 ceph-mon[23181]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:13 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:13.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:13.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:15.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:15 smithi026 ceph-mon[22362]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:15.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:15 smithi026 ceph-mon[27123]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:15 smithi046 ceph-mon[23181]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:42:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:42:15] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-04-30T09:42:16.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-30T09:42:16.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:16 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:16.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:16 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:16 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:17.942 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:17 smithi026 ceph-mon[27123]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:17.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:17 smithi026 ceph-mon[22362]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:18.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:17 smithi046 ceph-mon[23181]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:18.839 INFO:teuthology.orchestra.run.smithi026.stdout:Scheduled to redeploy mgr.y on host 'smithi026' 2024-04-30T09:42:18.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:18 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4160034195' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-30T09:42:18.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:18 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4160034195' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-30T09:42:18.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:18 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4160034195' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-30T09:42:18.989 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:19.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps --refresh' 2024-04-30T09:42:20.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[27123]: from='client.25096 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:42:20.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:20.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[27123]: Schedule redeploy daemon mgr.y 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[27123]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[22362]: from='client.25096 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[22362]: Schedule redeploy daemon mgr.y 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:42:20.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:19 smithi026 ceph-mon[22362]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-30T09:42:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:19 smithi046 ceph-mon[23181]: from='client.25096 -' entity='client.admin' cmd=[{"prefix": "orch daemon redeploy", "name": "mgr.y", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:42:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:19 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:19 smithi046 ceph-mon[23181]: Schedule redeploy daemon mgr.y 2024-04-30T09:42:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:19 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:19 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:19 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:42:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:19 smithi046 ceph-mon[23181]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-30T09:42:22.114 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:21 smithi026 ceph-mon[27123]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-30T09:42:22.114 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:21 smithi026 ceph-mon[22362]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-30T09:42:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:21 smithi046 ceph-mon[23181]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-30T09:42:22.797 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:42:22.797 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 79s ago 10m 16.8M - 0.23.0 ba2b418f427c 1648d8f22b65 2024-04-30T09:42:22.797 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (2m) 78s ago 10m 42.4M - 8.3.5 dad864ee21e9 9bd49a225988 2024-04-30T09:42:22.797 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 running (2m) 79s ago 9m 50.3M - 3.5 e1d6a67b021e cedfc064691e 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283 running (5m) 78s ago 15m 511M - 19.0.0-3357-g3a873b0c c50cf5fa633a 53731c91d116 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (17m) 79s ago 17m 410M - 17.2.0 e1d6a67b021e 780f8f988439 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (17m) 79s ago 17m 58.9M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (15m) 78s ago 15m 45.2M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (15m) 79s ago 15m 49.7M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) 79s ago 11m 16.6M - 1.3.1 1dbe0e931976 b6af5622fe4c 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (2m) 78s ago 11m 8774k - 1.3.1 1dbe0e931976 23b6576e6bb5 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (15m) 79s ago 15m 51.9M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (14m) 79s ago 14m 50.8M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (14m) 79s ago 14m 50.2M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (13m) 79s ago 13m 50.1M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (13m) 78s ago 13m 53.6M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:42:22.798 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (13m) 78s ago 13m 50.1M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:42:22.799 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (12m) 78s ago 12m 49.0M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:42:22.799 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (12m) 78s ago 12m 51.3M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:42:22.799 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (2m) 78s ago 11m 60.3M - 2.33.4 514e6a882f6e 1b17cf12c30b 2024-04-30T09:42:22.799 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (10m) 79s ago 10m 71.9M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:42:22.799 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (10m) 78s ago 10m 71.7M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:42:22.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:22.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:22.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:42:22.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:22.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:42:22.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:42:22.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:42:22.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:22.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[27123]: Deploying daemon mgr.y on smithi026 2024-04-30T09:42:22.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:22.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:22.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:42:22.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:22.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:42:22.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:42:22.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:42:22.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:22.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:22 smithi026 ceph-mon[22362]: Deploying daemon mgr.y on smithi026 2024-04-30T09:42:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:22 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:22 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:22 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:42:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:22 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:22 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:42:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:22 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:42:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:22 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:42:23.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:22 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:23.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:22 smithi046 ceph-mon[23181]: Deploying daemon mgr.y on smithi026 2024-04-30T09:42:23.796 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:24.043 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:23 smithi026 ceph-mon[22362]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:42:24.044 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:23 smithi026 ceph-mon[22362]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-30T09:42:24.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:23 smithi026 ceph-mon[27123]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:42:24.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:23 smithi026 ceph-mon[27123]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-30T09:42:24.171 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'sleep 180' 2024-04-30T09:42:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:23 smithi046 ceph-mon[23181]: from='client.25102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "refresh": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:42:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:23 smithi046 ceph-mon[23181]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 73 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-30T09:42:24.432 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:24 smithi026 systemd[1]: Stopping Ceph mgr.y for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:42:24.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:24 smithi026 podman[60262]: 2024-04-30 09:42:24.431251099 +0000 UTC m=+0.134729323 container died 780f8f988439fad6b67ea8ab1c9475d152b52e0f13596cd69eb996f6c90c5064 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, distribution-scope=public, vcs-type=git, io.openshift.tags=base centos centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., RELEASE=HEAD, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, maintainer=Guillaume Abrioux , description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.openshift.expose-services=, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True, version=8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754) 2024-04-30T09:42:25.018 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:24 smithi026 podman[60262]: 2024-04-30 09:42:24.825603041 +0000 UTC m=+0.529081302 container cleanup 780f8f988439fad6b67ea8ab1c9475d152b52e0f13596cd69eb996f6c90c5064 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, io.openshift.expose-services=, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, ceph=True, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, name=centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, maintainer=Guillaume Abrioux , summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vcs-type=git, release=754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True) 2024-04-30T09:42:25.019 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:24 smithi026 bash[60262]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y 2024-04-30T09:42:25.275 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:25 smithi026 ceph-mon[27123]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-30T09:42:25.275 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:25 smithi026 ceph-mon[22362]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-30T09:42:25.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:25 smithi026 podman[60302]: 2024-04-30 09:42:25.01834857 +0000 UTC m=+0.588433403 container remove 780f8f988439fad6b67ea8ab1c9475d152b52e0f13596cd69eb996f6c90c5064 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, build-date=2022-05-03T08:36:31.336870, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, vcs-type=git, architecture=x86_64, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vendor=Red Hat, Inc., GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, io.openshift.tags=base centos centos-stream) 2024-04-30T09:42:25.276 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:25 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-30T09:42:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:25 smithi046 ceph-mon[23181]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-30T09:42:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:42:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:42:25] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-04-30T09:42:26.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:26 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-30T09:42:26.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:26 smithi026 systemd[1]: Stopped Ceph mgr.y for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:42:26.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:26 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.y.service: Consumed 1min 3.050s CPU time. 2024-04-30T09:42:26.766 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:26 smithi026 systemd[1]: Starting Ceph mgr.y for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:42:27.024 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:26 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:27.025 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:26 smithi026 podman[60479]: 2024-04-30 09:42:26.821834014 +0000 UTC m=+0.114880286 container create e88d7265bd5b4866a015ab9005b08085602ae231f5297df4d8d43efaefa9b4b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD) 2024-04-30T09:42:27.025 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:26 smithi026 podman[60479]: 2024-04-30 09:42:26.725308873 +0000 UTC m=+0.018355146 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:42:27.025 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:26 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:26 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:27.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:27 smithi026 podman[60479]: 2024-04-30 09:42:27.024286938 +0000 UTC m=+0.317333210 container init e88d7265bd5b4866a015ab9005b08085602ae231f5297df4d8d43efaefa9b4b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T09:42:27.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:27 smithi026 podman[60479]: 2024-04-30 09:42:27.027017227 +0000 UTC m=+0.320063502 container start e88d7265bd5b4866a015ab9005b08085602ae231f5297df4d8d43efaefa9b4b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-30T09:42:27.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mgr[60498]: -- 172.21.15.26:0/1960737601 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x555d3a0525a0 con 0x555d3a027000 2024-04-30T09:42:27.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:27 smithi026 bash[60479]: e88d7265bd5b4866a015ab9005b08085602ae231f5297df4d8d43efaefa9b4b4 2024-04-30T09:42:27.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:27 smithi026 systemd[1]: Started Ceph mgr.y for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:42:28.094 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:27 smithi046 ceph-mon[23181]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-30T09:42:28.094 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:27 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.094 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:27 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.094 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:27 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.094 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:27 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.094 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:27 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:42:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[27123]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-30T09:42:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:42:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[22362]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 0 B/s wr, 119 op/s 2024-04-30T09:42:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:28.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:27 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:42:28.925 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:28 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:28.925 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:29.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:28 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:28 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:29.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:29 smithi026 ceph-mon[27123]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-30T09:42:29.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:29 smithi026 ceph-mon[22362]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-30T09:42:29.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:29 smithi046 ceph-mon[23181]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 0 B/s wr, 120 op/s 2024-04-30T09:42:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[27123]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-30T09:42:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:32.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:42:32.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[22362]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-30T09:42:32.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:32.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:42:32.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:31 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:31 smithi046 ceph-mon[23181]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-30T09:42:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:31 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:31 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:31 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:31 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:31 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:42:32.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:31 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:42:32.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:31 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:42:33.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:33 smithi026 ceph-mon[27123]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-30T09:42:33.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:33 smithi026 ceph-mon[22362]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-30T09:42:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:33 smithi046 ceph-mon[23181]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-30T09:42:33.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:34.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[27123]: Standby manager daemon y restarted 2024-04-30T09:42:34.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[27123]: Standby manager daemon y started 2024-04-30T09:42:34.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-30T09:42:34.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:42:34.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-30T09:42:34.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:42:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[22362]: Standby manager daemon y restarted 2024-04-30T09:42:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[22362]: Standby manager daemon y started 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:42:34] ENGINE Bus STARTING 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: CherryPy Checker: 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: The Application mounted at '' has an empty config. 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:42:34] ENGINE Serving on http://:::9283 2024-04-30T09:42:34.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:42:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:42:34] ENGINE Bus STARTED 2024-04-30T09:42:34.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:34 smithi046 ceph-mon[23181]: Standby manager daemon y restarted 2024-04-30T09:42:34.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:34 smithi046 ceph-mon[23181]: Standby manager daemon y started 2024-04-30T09:42:34.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:34 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-30T09:42:34.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:34 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:42:34.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:34 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-30T09:42:34.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:34 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:42:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:42:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:42:35] "GET /metrics HTTP/1.1" 200 37887 "" "Prometheus/2.33.4" 2024-04-30T09:42:35.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:35 smithi026 ceph-mon[22362]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-30T09:42:35.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:35 smithi026 ceph-mon[22362]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-30T09:42:35.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:35 smithi026 ceph-mon[27123]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-30T09:42:35.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:35 smithi026 ceph-mon[27123]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-30T09:42:35.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:35 smithi046 ceph-mon[23181]: mgrmap e27: x(active, since 3m), standbys: y 2024-04-30T09:42:35.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:35 smithi046 ceph-mon[23181]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 0 B/s wr, 113 op/s 2024-04-30T09:42:37.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:36 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:37.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:36 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:36 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:38.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:37 smithi026 ceph-mon[22362]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:37 smithi026 ceph-mon[27123]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:37 smithi046 ceph-mon[23181]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:38.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:39.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:39 smithi026 ceph-mon[27123]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:39.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:39 smithi026 ceph-mon[22362]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:39 smithi046 ceph-mon[23181]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:42.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:41 smithi026 ceph-mon[22362]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:42.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:41 smithi026 ceph-mon[27123]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:41 smithi046 ceph-mon[23181]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:43.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:43 smithi026 ceph-mon[27123]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:43.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:43 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:43 smithi026 ceph-mon[22362]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:43 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:43 smithi046 ceph-mon[23181]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:43 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:43.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:43 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:43.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:45.171 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:45 smithi046 ceph-mon[23181]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:45.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:45 smithi026 ceph-mon[27123]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:45.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:45 smithi026 ceph-mon[22362]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:45.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:42:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:42:45] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-04-30T09:42:46.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:46 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:46.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:46 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:46 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:48.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:47 smithi026 ceph-mon[27123]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:47 smithi026 ceph-mon[22362]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:48.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:47 smithi046 ceph-mon[23181]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:48.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:49.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:49 smithi026 ceph-mon[22362]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:49 smithi026 ceph-mon[27123]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:49 smithi046 ceph-mon[23181]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:52.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:51 smithi026 ceph-mon[22362]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:52.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:51 smithi026 ceph-mon[27123]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:51 smithi046 ceph-mon[23181]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:53.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:53 smithi026 ceph-mon[22362]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:53 smithi026 ceph-mon[27123]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:53 smithi046 ceph-mon[23181]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:53.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:55.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:55 smithi026 ceph-mon[27123]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:55.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:55 smithi026 ceph-mon[22362]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:55 smithi046 ceph-mon[23181]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:42:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:42:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:42:55] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-04-30T09:42:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:56 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:56 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:56 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:42:58.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:57 smithi026 ceph-mon[27123]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:58.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:57 smithi026 ceph-mon[22362]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:57 smithi046 ceph-mon[23181]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:42:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:58 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:58.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:42:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:42:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:42:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:58 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:42:59.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:58 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:00.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:42:59 smithi026 ceph-mon[27123]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:00.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:42:59 smithi026 ceph-mon[22362]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:42:59 smithi046 ceph-mon[23181]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:02.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:01 smithi026 ceph-mon[27123]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:02.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:01 smithi026 ceph-mon[22362]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:01 smithi046 ceph-mon[23181]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:03.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:03 smithi026 ceph-mon[27123]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:03.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:03 smithi026 ceph-mon[22362]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:03 smithi046 ceph-mon[23181]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:03.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:03.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:05.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:05 smithi026 ceph-mon[27123]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:05.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:05 smithi026 ceph-mon[22362]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:43:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:43:05] "GET /metrics HTTP/1.1" 200 37885 "" "Prometheus/2.33.4" 2024-04-30T09:43:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:05 smithi046 ceph-mon[23181]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:07.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:06 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:07.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:06 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:06 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:08.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:07 smithi026 ceph-mon[22362]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:08.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:07 smithi026 ceph-mon[27123]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:07 smithi046 ceph-mon[23181]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:08.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:09.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:09 smithi026 ceph-mon[27123]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:09.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:09 smithi026 ceph-mon[22362]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:09 smithi046 ceph-mon[23181]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:12.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:11 smithi026 ceph-mon[27123]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:43:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:11 smithi026 ceph-mon[22362]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:43:12.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:11 smithi046 ceph-mon[23181]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:43:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:13 smithi026 ceph-mon[27123]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:43:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:13 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:13 smithi026 ceph-mon[22362]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:43:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:13 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:13 smithi046 ceph-mon[23181]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:43:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:13 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:13.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:13.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:15 smithi026 ceph-mon[27123]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:15 smithi026 ceph-mon[22362]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:15.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:15 smithi046 ceph-mon[23181]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:43:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:43:15] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-04-30T09:43:17.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:16 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:17.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:16 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:16 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:18.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:17 smithi026 ceph-mon[27123]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:43:18.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:17 smithi026 ceph-mon[22362]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:43:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:17 smithi046 ceph-mon[23181]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:43:18.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:18.660Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:19.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:19 smithi026 ceph-mon[27123]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:19 smithi026 ceph-mon[22362]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:19 smithi046 ceph-mon[23181]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:22.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:21 smithi026 ceph-mon[27123]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:21 smithi026 ceph-mon[22362]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:21 smithi046 ceph-mon[23181]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:23 smithi026 ceph-mon[27123]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:23 smithi026 ceph-mon[22362]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:23.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:23 smithi046 ceph-mon[23181]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:23.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:23.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:25.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:25 smithi026 ceph-mon[22362]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:25.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:25 smithi026 ceph-mon[27123]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:25.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:25 smithi046 ceph-mon[23181]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:25.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:43:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:43:25] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-04-30T09:43:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:26 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:27.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:26 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:26 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:27 smithi026 ceph-mon[27123]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:27 smithi026 ceph-mon[22362]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:27 smithi046 ceph-mon[23181]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:28 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:28.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:28 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:28 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:30.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:29 smithi026 ceph-mon[27123]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:30.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:29 smithi026 ceph-mon[22362]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:29 smithi046 ceph-mon[23181]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:32.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:31 smithi026 ceph-mon[27123]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:32.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:31 smithi026 ceph-mon[22362]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:31 smithi046 ceph-mon[23181]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:33.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:32 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:43:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:32 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:43:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:32 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:43:33.792 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:33.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:33 smithi026 ceph-mon[22362]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:33 smithi026 ceph-mon[27123]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:33 smithi046 ceph-mon[23181]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:35.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:35 smithi026 ceph-mon[27123]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:35.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:35 smithi026 ceph-mon[22362]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:43:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:43:35] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-04-30T09:43:35.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:35 smithi046 ceph-mon[23181]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:36 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:36 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:43:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:36 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:43:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:36 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:36 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:43:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:36 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:43:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:36 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:36 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:43:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:36 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:43:38.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:37 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:43:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:37 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:43:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:37 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:43:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:37 smithi026 ceph-mon[27123]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:37 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:43:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:37 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:43:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:37 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:43:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:37 smithi026 ceph-mon[22362]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:37 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:43:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:37 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:43:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:37 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:43:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:37 smithi046 ceph-mon[23181]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:38.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:38 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:38.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:39.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:39 smithi026 ceph-mon[22362]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:39.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:39 smithi026 ceph-mon[27123]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:39.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:39 smithi046 ceph-mon[23181]: pgmap v143: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:41 smithi026 ceph-mon[27123]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:42.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:41 smithi026 ceph-mon[22362]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:41 smithi046 ceph-mon[23181]: pgmap v144: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:43.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:43 smithi026 ceph-mon[27123]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:43.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:43 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:43 smithi026 ceph-mon[22362]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:43 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:43.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:43 smithi046 ceph-mon[23181]: pgmap v145: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:43 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:43.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:43 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:43.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:45.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:45 smithi026 ceph-mon[27123]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:45.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:45 smithi026 ceph-mon[22362]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:45.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:45 smithi046 ceph-mon[23181]: pgmap v146: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:45.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:43:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:43:45] "GET /metrics HTTP/1.1" 200 37884 "" "Prometheus/2.33.4" 2024-04-30T09:43:47.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:46 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:46 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:46 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:48.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:47 smithi026 ceph-mon[22362]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:48.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:47 smithi026 ceph-mon[27123]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:48.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:47 smithi046 ceph-mon[23181]: pgmap v147: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:48.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:48.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:49.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:49 smithi026 ceph-mon[22362]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:49 smithi026 ceph-mon[27123]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:49 smithi046 ceph-mon[23181]: pgmap v148: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:52.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:51 smithi026 ceph-mon[27123]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:51 smithi026 ceph-mon[22362]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:51 smithi046 ceph-mon[23181]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:53.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:53 smithi026 ceph-mon[22362]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:53 smithi026 ceph-mon[27123]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:53 smithi046 ceph-mon[23181]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:53.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:53.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:55.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:55 smithi026 ceph-mon[27123]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:55.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:55 smithi026 ceph-mon[22362]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:55 smithi046 ceph-mon[23181]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:43:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:43:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:43:55] "GET /metrics HTTP/1.1" 200 37884 "" "Prometheus/2.33.4" 2024-04-30T09:43:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:56 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:57.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:56 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:56 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:43:58.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:57 smithi026 ceph-mon[27123]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:58.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:57 smithi026 ceph-mon[22362]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:58.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:57 smithi046 ceph-mon[23181]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:43:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:58 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:58.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:43:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:43:58.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:43:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:58 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:43:59.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:58 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:00.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:43:59 smithi026 ceph-mon[27123]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:43:59 smithi026 ceph-mon[22362]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:43:59 smithi046 ceph-mon[23181]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:02.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:01 smithi026 ceph-mon[27123]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:02.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:01 smithi026 ceph-mon[22362]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:01 smithi046 ceph-mon[23181]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:03.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:03 smithi026 ceph-mon[27123]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:03.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:03 smithi026 ceph-mon[22362]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:03 smithi046 ceph-mon[23181]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:03.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:44:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:44:03.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:44:05.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:05 smithi026 ceph-mon[27123]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:05 smithi026 ceph-mon[22362]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:05 smithi046 ceph-mon[23181]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:44:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:44:05] "GET /metrics HTTP/1.1" 200 37891 "" "Prometheus/2.33.4" 2024-04-30T09:44:07.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:06 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:07.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:06 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:06 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:08.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:07 smithi026 ceph-mon[27123]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:08.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:07 smithi026 ceph-mon[22362]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:07 smithi046 ceph-mon[23181]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:08.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:44:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:44:08.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:44:09.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:09 smithi026 ceph-mon[27123]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:09.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:09 smithi026 ceph-mon[22362]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:09 smithi046 ceph-mon[23181]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:12.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:11 smithi026 ceph-mon[27123]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:11 smithi026 ceph-mon[22362]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:11 smithi046 ceph-mon[23181]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:13 smithi026 ceph-mon[27123]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:13 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:13 smithi026 ceph-mon[22362]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:13 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:13 smithi046 ceph-mon[23181]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:13 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:13.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:44:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:44:13.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:44:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:15 smithi026 ceph-mon[27123]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:15 smithi026 ceph-mon[22362]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:15 smithi046 ceph-mon[23181]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:44:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:44:15] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-04-30T09:44:17.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:16 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:17.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:16 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:16 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:18.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:17 smithi026 ceph-mon[27123]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:18.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:17 smithi026 ceph-mon[22362]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:17 smithi046 ceph-mon[23181]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:18.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:44:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:44:18.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:44:19.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:19 smithi026 ceph-mon[27123]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:19.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:19 smithi026 ceph-mon[22362]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:19 smithi046 ceph-mon[23181]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:22.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:21 smithi026 ceph-mon[27123]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:21 smithi026 ceph-mon[22362]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:21 smithi046 ceph-mon[23181]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:23 smithi026 ceph-mon[27123]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:23 smithi026 ceph-mon[22362]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:23 smithi046 ceph-mon[23181]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:23.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:44:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:44:23.506Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:44:25.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:25 smithi026 ceph-mon[27123]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:25.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:25 smithi026 ceph-mon[22362]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:25 smithi046 ceph-mon[23181]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:44:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:44:25] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-04-30T09:44:27.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:26 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:27.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:26 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:26 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:27 smithi026 ceph-mon[27123]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:28.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:27 smithi026 ceph-mon[22362]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:27 smithi046 ceph-mon[23181]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:28 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:28.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:44:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:44:28.659Z caller=manager.go:609 level=warn component="rule manager" group=osd msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on(ceph_daemon) group_left(hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"172.21.15.46:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:44:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:28 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:29.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:28 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:30.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:29 smithi026 ceph-mon[27123]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:29 smithi026 ceph-mon[22362]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:29 smithi046 ceph-mon[23181]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:32.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:31 smithi026 ceph-mon[22362]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:31 smithi026 ceph-mon[27123]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:31 smithi046 ceph-mon[23181]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:33.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:33 smithi026 ceph-mon[22362]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:33.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:33 smithi026 ceph-mon[27123]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:33 smithi046 ceph-mon[23181]: pgmap v170: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:33.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:44:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:44:33.505Z caller=manager.go:609 level=warn component="rule manager" group=nodes msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on(instance) group_left(nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi026\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"703f4aea-06d3-11ef-bc93-c7b262605968\", domainname=\"(none)\", instance=\"smithi026\", job=\"node\", machine=\"x86_64\", nodename=\"smithi026\", release=\"5.14.0-437.el9.x86_64\", sysname=\"Linux\", version=\"#1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-30T09:44:35.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:35 smithi026 ceph-mon[22362]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:35.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:35 smithi026 ceph-mon[27123]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:35.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:35 smithi046 ceph-mon[23181]: pgmap v171: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:44:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:44:35] "GET /metrics HTTP/1.1" 200 37885 "" "Prometheus/2.33.4" 2024-04-30T09:44:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:36 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:37.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:36 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:37.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:36 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:38.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:44:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[27123]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:44:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:44:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:44:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:44:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[22362]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:44:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:44:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:37 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:44:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:37 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:44:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:37 smithi046 ceph-mon[23181]: pgmap v172: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:37 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:44:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:37 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:44:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:37 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:44:39.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:39 smithi026 ceph-mon[27123]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:39.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:39 smithi026 ceph-mon[22362]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:39 smithi046 ceph-mon[23181]: pgmap v173: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:41 smithi026 ceph-mon[27123]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:41 smithi026 ceph-mon[22362]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:42.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:41 smithi046 ceph-mon[23181]: pgmap v174: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:43.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:43 smithi026 ceph-mon[27123]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:43.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:43 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:43 smithi026 ceph-mon[22362]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:43 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:43 smithi046 ceph-mon[23181]: pgmap v175: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:43 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:45.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:45 smithi026 ceph-mon[27123]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:45.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:45 smithi026 ceph-mon[22362]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:45.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:45 smithi046 ceph-mon[23181]: pgmap v176: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:45.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:44:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:44:45] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-04-30T09:44:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:46 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:46 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:46 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:48.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:47 smithi026 ceph-mon[22362]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:48.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:47 smithi026 ceph-mon[27123]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:47 smithi046 ceph-mon[23181]: pgmap v177: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:49.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:49 smithi026 ceph-mon[22362]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:49 smithi026 ceph-mon[27123]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:49 smithi046 ceph-mon[23181]: pgmap v178: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:52.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:51 smithi026 ceph-mon[22362]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:52.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:51 smithi026 ceph-mon[27123]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:52.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:51 smithi046 ceph-mon[23181]: pgmap v179: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:53.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:53 smithi026 ceph-mon[27123]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:53 smithi026 ceph-mon[22362]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:53 smithi046 ceph-mon[23181]: pgmap v180: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:55.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:55 smithi026 ceph-mon[27123]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:55.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:55 smithi026 ceph-mon[22362]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:55 smithi046 ceph-mon[23181]: pgmap v181: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:44:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:44:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:44:55] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-04-30T09:44:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:56 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:56 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:57.233 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:56 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:44:58.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:57 smithi026 ceph-mon[27123]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:58.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:57 smithi026 ceph-mon[22362]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:57 smithi046 ceph-mon[23181]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:44:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:58 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:59.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:58 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:44:59.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:58 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:00.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:44:59 smithi026 ceph-mon[27123]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:44:59 smithi026 ceph-mon[22362]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:44:59 smithi046 ceph-mon[23181]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:02.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:01 smithi026 ceph-mon[27123]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:02.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:01 smithi026 ceph-mon[22362]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:01 smithi046 ceph-mon[23181]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:03.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:03 smithi026 ceph-mon[27123]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:03.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:03 smithi026 ceph-mon[22362]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:03 smithi046 ceph-mon[23181]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:05.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:05 smithi026 ceph-mon[27123]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:05 smithi026 ceph-mon[22362]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:05 smithi046 ceph-mon[23181]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:05.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:45:05] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-04-30T09:45:07.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:06 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:07.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:06 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:06 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:08.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:07 smithi026 ceph-mon[27123]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:08.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:07 smithi026 ceph-mon[22362]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:07 smithi046 ceph-mon[23181]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:09.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:09 smithi026 ceph-mon[22362]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:09.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:09 smithi026 ceph-mon[27123]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:09 smithi046 ceph-mon[23181]: pgmap v188: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:12.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:11 smithi026 ceph-mon[27123]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:11 smithi026 ceph-mon[22362]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:11 smithi046 ceph-mon[23181]: pgmap v189: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:13.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:13 smithi026 ceph-mon[22362]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:13 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:13 smithi026 ceph-mon[27123]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:13 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:13 smithi046 ceph-mon[23181]: pgmap v190: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:13 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:15.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:15 smithi026 ceph-mon[22362]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:15.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:15 smithi026 ceph-mon[27123]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:15 smithi046 ceph-mon[23181]: pgmap v191: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:45:15] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-04-30T09:45:17.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:16 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:17.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:16 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:16 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:18.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:17 smithi026 ceph-mon[22362]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:18.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:17 smithi026 ceph-mon[27123]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:17 smithi046 ceph-mon[23181]: pgmap v192: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:19.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:19 smithi026 ceph-mon[27123]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:19 smithi026 ceph-mon[22362]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:19 smithi046 ceph-mon[23181]: pgmap v193: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:22.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:21 smithi026 ceph-mon[27123]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:21 smithi026 ceph-mon[22362]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:21 smithi046 ceph-mon[23181]: pgmap v194: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:23.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:23 smithi026 ceph-mon[22362]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:23.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:23 smithi026 ceph-mon[27123]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:23 smithi046 ceph-mon[23181]: pgmap v195: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:25.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:25 smithi026 ceph-mon[27123]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:25.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:25 smithi026 ceph-mon[22362]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:25 smithi046 ceph-mon[23181]: pgmap v196: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:45:25] "GET /metrics HTTP/1.1" 200 37889 "" "Prometheus/2.33.4" 2024-04-30T09:45:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:26 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:26 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:26 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:27.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T09:45:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:27 smithi026 ceph-mon[27123]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:27 smithi026 ceph-mon[22362]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:27 smithi046 ceph-mon[23181]: pgmap v197: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:28 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:28 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:29.206 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:45:29.206 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (5m) 112s ago 13m 23.8M - 0.23.0 ba2b418f427c 1648d8f22b65 2024-04-30T09:45:29.206 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (5m) 2m ago 13m 42.4M - 8.3.5 dad864ee21e9 9bd49a225988 2024-04-30T09:45:29.206 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 running (5m) 112s ago 12m 55.2M - 3.5 e1d6a67b021e cedfc064691e 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283 running (8m) 2m ago 18m 514M - 19.0.0-3357-g3a873b0c c50cf5fa633a 53731c91d116 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (3m) 112s ago 20m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (20m) 112s ago 20m 59.1M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (18m) 2m ago 18m 40.1M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (19m) 112s ago 19m 43.4M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) 112s ago 14m 18.0M - 1.3.1 1dbe0e931976 b6af5622fe4c 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (5m) 2m ago 14m 9303k - 1.3.1 1dbe0e931976 23b6576e6bb5 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (18m) 112s ago 18m 52.5M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (17m) 112s ago 17m 52.0M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (17m) 112s ago 17m 52.0M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (17m) 112s ago 17m 50.1M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (16m) 2m ago 16m 54.4M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (16m) 2m ago 16m 51.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (15m) 2m ago 15m 50.2M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:45:29.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (15m) 2m ago 15m 52.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:45:29.208 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (5m) 2m ago 14m 71.4M - 2.33.4 514e6a882f6e 1b17cf12c30b 2024-04-30T09:45:29.208 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (13m) 112s ago 13m 72.7M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:45:29.208 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (13m) 2m ago 13m 72.0M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:45:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:28 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:29.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions' 2024-04-30T09:45:29.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:29 smithi026 ceph-mon[27123]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:29.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:29 smithi026 ceph-mon[22362]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:29 smithi046 ceph-mon[23181]: pgmap v198: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:31.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:30 smithi026 ceph-mon[22362]: from='client.15216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:45:31.151 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:30 smithi026 ceph-mon[27123]: from='client.15216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:45:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:30 smithi046 ceph-mon[23181]: from='client.15216 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:45:31.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:45:31.950 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:45:31.950 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:45:31.950 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:45:31.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:45:31.950 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:45:31.950 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-30T09:45:31.950 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:45:31.950 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:45:31.950 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:45:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:31 smithi026 ceph-mon[27123]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:31 smithi026 ceph-mon[22362]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:31 smithi046 ceph-mon[23181]: pgmap v199: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:32.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T09:45:33.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:32 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/407687974' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:45:33.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:32 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/407687974' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:45:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:32 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/407687974' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:45:34.144 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:33 smithi026 ceph-mon[27123]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:34.144 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:33 smithi026 ceph-mon[22362]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:33 smithi046 ceph-mon[23181]: pgmap v200: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:34.664 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-30T09:45:34.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:34 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/26219576' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:45:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:34 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/26219576' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:45:35.172 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:34 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/26219576' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:45:35.281 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph -s' 2024-04-30T09:45:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:45:35] "GET /metrics HTTP/1.1" 200 37885 "" "Prometheus/2.33.4" 2024-04-30T09:45:35.903 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:35 smithi026 ceph-mon[27123]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:35.903 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:35 smithi026 ceph-mon[22362]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:35 smithi046 ceph-mon[23181]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:36 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:36 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:36 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: cluster: 2024-04-30T09:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: id: 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: health: HEALTH_OK 2024-04-30T09:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: services: 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: mon: 3 daemons, quorum a,c,b (age 18m) 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: mgr: x(active, since 6m), standbys: y 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: osd: 8 osds: 8 up (since 15m), 8 in (since 15m) 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: data: 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: pools: 6 pools, 161 pgs 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: objects: 209 objects, 585 KiB 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: pgs: 161 active+clean 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: io: 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: client: 1.2 KiB/s rd, 1 op/s rd, 0 op/s wr 2024-04-30T09:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:45:37.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:37 smithi026 ceph-mon[22362]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:37.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:37 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2678074177' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:45:37.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:37 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:45:37.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:37 smithi026 ceph-mon[27123]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:37.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:37 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2678074177' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:45:37.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:37 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:45:38.003 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph mgr fail' 2024-04-30T09:45:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:37 smithi046 ceph-mon[23181]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:45:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:37 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2678074177' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:45:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:37 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:45:38.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:38 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:45:38.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:38 smithi026 ceph-mon[22362]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:45:38.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:38 smithi026 ceph-mon[22362]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:45:38.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:38 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:45:38.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:38 smithi026 ceph-mon[27123]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:45:38.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:38 smithi026 ceph-mon[27123]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:45:39.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:38 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:45:39.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:38 smithi046 ceph-mon[23181]: from='mgr.24797 172.21.15.46:0/2020213754' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:45:39.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:38 smithi046 ceph-mon[23181]: from='mgr.24797 ' entity='mgr.x' 2024-04-30T09:45:40.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:39 smithi026 ceph-mon[27123]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:39 smithi026 ceph-mon[22362]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:39 smithi046 ceph-mon[23181]: pgmap v203: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:45:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2197585697' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:45:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-mon[27123]: osdmap e104: 8 total, 8 up, 8 in 2024-04-30T09:45:41.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-mon[27123]: Standby manager daemon x started 2024-04-30T09:45:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2197585697' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:45:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-mon[22362]: osdmap e104: 8 total, 8 up, 8 in 2024-04-30T09:45:41.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-mon[22362]: Standby manager daemon x started 2024-04-30T09:45:41.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:40] ENGINE Bus STOPPING 2024-04-30T09:45:41.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:40] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:45:41.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:40] ENGINE Bus STOPPED 2024-04-30T09:45:41.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:40 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:40] ENGINE Bus STARTING 2024-04-30T09:45:41.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:41] ENGINE Serving on http://:::9283 2024-04-30T09:45:41.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:41] ENGINE Bus STARTED 2024-04-30T09:45:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:40 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2197585697' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:45:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:40 smithi046 ceph-mon[23181]: osdmap e104: 8 total, 8 up, 8 in 2024-04-30T09:45:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:40 smithi046 ceph-mon[23181]: Standby manager daemon x started 2024-04-30T09:45:41.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:45:40.859+0000 7f795bf3e700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-30T09:45:41.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ignoring --setuser ceph since I am not root 2024-04-30T09:45:41.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:40 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ignoring --setgroup ceph since I am not root 2024-04-30T09:45:41.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:40 smithi046 ceph-mgr[43498]: -- 172.21.15.46:0/2360436239 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5596859a85a0 con 0x55968597cc00 2024-04-30T09:45:41.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'sleep 180' 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2197585697' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: mgrmap e28: y(active, starting, since 0.786547s), standbys: x 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:45:42.124 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: Manager daemon y is now available 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:45:42.125 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:45:42.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2197585697' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-30T09:45:42.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: mgrmap e28: y(active, starting, since 0.786547s), standbys: x 2024-04-30T09:45:42.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:45:42.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:45:42.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:45:42.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:45:42.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:45:42.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:45:42.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: Manager daemon y is now available 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2197585697' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: mgrmap e28: y(active, starting, since 0.786547s), standbys: x 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:45:42.136 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: Manager daemon y is now available 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:45:42.137 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:45:42.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[27123]: [30/Apr/2024:09:45:41] ENGINE Bus STARTING 2024-04-30T09:45:42.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[27123]: [30/Apr/2024:09:45:41] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:45:42.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[27123]: [30/Apr/2024:09:45:41] ENGINE Serving on http://172.21.15.26:8765 2024-04-30T09:45:42.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[27123]: [30/Apr/2024:09:45:41] ENGINE Bus STARTED 2024-04-30T09:45:42.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[27123]: mgrmap e29: y(active, since 1.79256s), standbys: x 2024-04-30T09:45:42.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[27123]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:42.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[22362]: [30/Apr/2024:09:45:41] ENGINE Bus STARTING 2024-04-30T09:45:42.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[22362]: [30/Apr/2024:09:45:41] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:45:42.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[22362]: [30/Apr/2024:09:45:41] ENGINE Serving on http://172.21.15.26:8765 2024-04-30T09:45:42.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[22362]: [30/Apr/2024:09:45:41] ENGINE Bus STARTED 2024-04-30T09:45:42.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[22362]: mgrmap e29: y(active, since 1.79256s), standbys: x 2024-04-30T09:45:42.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:42 smithi026 ceph-mon[22362]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:42 smithi046 ceph-mon[23181]: [30/Apr/2024:09:45:41] ENGINE Bus STARTING 2024-04-30T09:45:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:42 smithi046 ceph-mon[23181]: [30/Apr/2024:09:45:41] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:45:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:42 smithi046 ceph-mon[23181]: [30/Apr/2024:09:45:41] ENGINE Serving on http://172.21.15.26:8765 2024-04-30T09:45:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:42 smithi046 ceph-mon[23181]: [30/Apr/2024:09:45:41] ENGINE Bus STARTED 2024-04-30T09:45:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:42 smithi046 ceph-mon[23181]: mgrmap e29: y(active, since 1.79256s), standbys: x 2024-04-30T09:45:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:42 smithi046 ceph-mon[23181]: pgmap v3: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:44.130 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:43 smithi046 ceph-mon[23181]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:44.130 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:43 smithi046 ceph-mon[23181]: mgrmap e30: y(active, since 2s), standbys: x 2024-04-30T09:45:44.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:43 smithi026 ceph-mon[27123]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:44.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:43 smithi026 ceph-mon[27123]: mgrmap e30: y(active, since 2s), standbys: x 2024-04-30T09:45:44.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:43 smithi026 ceph-mon[22362]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:44.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:43 smithi026 ceph-mon[22362]: mgrmap e30: y(active, since 2s), standbys: x 2024-04-30T09:45:45.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:45 smithi026 ceph-mon[27123]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:45.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:45 smithi026 ceph-mon[27123]: mgrmap e31: y(active, since 4s), standbys: x 2024-04-30T09:45:45.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:45 smithi026 ceph-mon[22362]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:45.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:45 smithi026 ceph-mon[22362]: mgrmap e31: y(active, since 4s), standbys: x 2024-04-30T09:45:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:45 smithi046 ceph-mon[23181]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:45:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:45 smithi046 ceph-mon[23181]: mgrmap e31: y(active, since 4s), standbys: x 2024-04-30T09:45:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[27123]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[22362]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:46 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:46 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:46 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:46 smithi046 ceph-mon[23181]: from='client.24926 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:45:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:46 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:46 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:47 smithi046 ceph-mon[23181]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:45:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:47 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:47 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:47 smithi026 ceph-mon[22362]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:45:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:47 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:47 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:48.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:47 smithi026 ceph-mon[27123]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:45:48.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:47 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:48.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:47 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:48.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:45:48] ENGINE Bus STARTING 2024-04-30T09:45:48.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: CherryPy Checker: 2024-04-30T09:45:48.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: The Application mounted at '' has an empty config. 2024-04-30T09:45:48.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:45:48.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:45:48] ENGINE Serving on http://:::9283 2024-04-30T09:45:48.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:48 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:45:48] ENGINE Bus STARTED 2024-04-30T09:45:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: Standby manager daemon x restarted 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: Standby manager daemon x started 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:45:49.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:45:49.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: Standby manager daemon x restarted 2024-04-30T09:45:49.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: Standby manager daemon x started 2024-04-30T09:45:49.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:45:49.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:45:49.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:45:49.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:49 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:45:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:45:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:45:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: Standby manager daemon x restarted 2024-04-30T09:45:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: Standby manager daemon x started 2024-04-30T09:45:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:45:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:45:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:45:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:49 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:45:50.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:45:50.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:45:50.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:45:50.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:45:50.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: mgrmap e32: y(active, since 9s), standbys: x 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:45:50.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:45:50.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:45:50.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:45:50.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:45:50.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:45:50.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:45:50.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:45:50.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:45:50.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:45:50.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:45:50.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: mgrmap e32: y(active, since 9s), standbys: x 2024-04-30T09:45:50.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:45:50.187 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:50 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:45:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:45:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:45:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:45:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:45:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:45:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:45:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: mgrmap e32: y(active, since 9s), standbys: x 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:45:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:50 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:45:51.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:51 smithi026 ceph-mon[27123]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:45:51.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:51 smithi026 ceph-mon[27123]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:45:51.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:51 smithi026 ceph-mon[27123]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:45:51.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:51 smithi026 ceph-mon[22362]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:45:51.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:51 smithi026 ceph-mon[22362]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:45:51.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:51 smithi026 ceph-mon[22362]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:45:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:51 smithi046 ceph-mon[23181]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:45:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:51 smithi046 ceph-mon[23181]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:45:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:51 smithi046 ceph-mon[23181]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:45:54.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:53 smithi026 ceph-mon[27123]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:45:54.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:53 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:54.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:53 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:54.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:53 smithi026 ceph-mon[22362]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:45:54.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:53 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:54.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:53 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:53 smithi046 ceph-mon[23181]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:45:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:53 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:53 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:54.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:54 smithi046 ceph-mon[23181]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:45:54.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:54 smithi046 ceph-mon[23181]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:45:54.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:54 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1346737233' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:45:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:54 smithi026 ceph-mon[27123]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:45:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:54 smithi026 ceph-mon[27123]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:45:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:54 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1346737233' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:45:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:54 smithi026 ceph-mon[22362]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:45:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:54 smithi026 ceph-mon[22362]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:45:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:54 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1346737233' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:45:55.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:45:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:45:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.33.4" 2024-04-30T09:45:56.117 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:55 smithi046 ceph-mon[23181]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:45:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:55 smithi026 ceph-mon[27123]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:45:56.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:55 smithi026 ceph-mon[22362]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:45:56.983 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:56 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:56 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:57.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:56 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:45:57.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 systemd[1]: Stopping Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:45:57.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 bash[52064]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:45:57.618 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.253Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-30T09:45:57.618 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.253Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-30T09:45:57.618 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.253Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.253Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.253Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.253Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.253Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.253Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.254Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.254Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.254Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[46574]: ts=2024-04-30T09:45:57.254Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-30T09:45:57.619 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 podman[52077]: 2024-04-30 09:45:57.263818163 +0000 UTC m=+0.117109891 container died 1b17cf12c30b4c9b4966720562e86995797e813d3eef733867f1fade97491095 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:45:57.905 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 podman[52077]: 2024-04-30 09:45:57.703463902 +0000 UTC m=+0.556755632 container cleanup 1b17cf12c30b4c9b4966720562e86995797e813d3eef733867f1fade97491095 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:45:57.905 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 bash[52077]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a 2024-04-30T09:45:58.159 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:57 smithi046 ceph-mon[23181]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:45:58.160 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 podman[52089]: 2024-04-30 09:45:57.904724145 +0000 UTC m=+0.638897982 container remove 1b17cf12c30b4c9b4966720562e86995797e813d3eef733867f1fade97491095 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:45:58.160 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 bash[52105]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:45:58.160 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-30T09:45:58.160 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 systemd[1]: Stopped Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:45:58.160 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 2.073s CPU time. 2024-04-30T09:45:58.160 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:57 smithi046 systemd[1]: Starting Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:45:58.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:57 smithi026 ceph-mon[27123]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:45:58.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:57 smithi026 ceph-mon[22362]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:45:58.430 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 podman[52173]: 2024-04-30 09:45:58.063246013 +0000 UTC m=+0.019848886 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-30T09:45:58.430 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 podman[52173]: 2024-04-30 09:45:58.1669045 +0000 UTC m=+0.123507403 container create a6e8333ff844491e379c59aa7eb433ff439815dfb82560502eedacec52b24346 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:45:58.430 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 podman[52173]: 2024-04-30 09:45:58.402523215 +0000 UTC m=+0.359126078 container init a6e8333ff844491e379c59aa7eb433ff439815dfb82560502eedacec52b24346 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:45:58.431 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 podman[52173]: 2024-04-30 09:45:58.405185406 +0000 UTC m=+0.361788267 container start a6e8333ff844491e379c59aa7eb433ff439815dfb82560502eedacec52b24346 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:45:58.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.430Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-30T09:45:58.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.430Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-30T09:45:58.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.430Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-30T09:45:58.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.430Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi046 (none))" 2024-04-30T09:45:58.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.430Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-30T09:45:58.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.430Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-30T09:45:58.735 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.431Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-30T09:45:58.735 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.432Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-30T09:45:58.735 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.433Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-30T09:45:58.735 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.435Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-30T09:45:58.735 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.435Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.622µs 2024-04-30T09:45:58.735 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.435Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-30T09:45:58.735 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 bash[52173]: a6e8333ff844491e379c59aa7eb433ff439815dfb82560502eedacec52b24346 2024-04-30T09:45:58.735 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 systemd[1]: Started Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:45:58.735 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.624Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-30T09:45:58.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:58 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:58] ENGINE Bus STOPPING 2024-04-30T09:45:58.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:58 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:45:58.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:58 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:58] ENGINE Bus STOPPED 2024-04-30T09:45:58.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:58 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:58] ENGINE Bus STARTING 2024-04-30T09:45:58.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:58 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:58] ENGINE Serving on http://:::9283 2024-04-30T09:45:58.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:45:58 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:45:58] ENGINE Bus STARTED 2024-04-30T09:45:59.158 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:58.887Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-30T09:45:59.158 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:59.129Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-30T09:45:59.158 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:59.129Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-30T09:45:59.158 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:59.129Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=52.484µs wal_replay_duration=694.034603ms total_replay_duration=694.107471ms 2024-04-30T09:45:59.158 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:59.134Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-30T09:45:59.158 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:59.134Z caller=main.go:947 level=info msg="TSDB started" 2024-04-30T09:45:59.158 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:59.134Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-30T09:45:59.158 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:59.157Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.788056ms db_storage=1.04µs remote_storage=1.206µs web_handler=349ns query_engine=1.02µs scrape=252.833µs scrape_sd=120.162µs notify=21.769µs notify_sd=18.24µs rules=21.997485ms 2024-04-30T09:45:59.158 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:45:59.157Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-30T09:45:59.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:59.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:59.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:45:59.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:45:59.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:45:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:45:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:45:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:45:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:45:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:45:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:45:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:45:59 smithi046 ceph-mon[23181]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:45:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:45:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:45:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:45:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:45:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:45:59.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:45:59.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:45:59.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:45:59.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:59.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:45:59.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[27123]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:45:59.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:59.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:59.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:45:59.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard iscsi-gateway-list"}]: dispatch 2024-04-30T09:45:59.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:45:59.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-iscsi-api-ssl-verification", "value": "true"}]: dispatch 2024-04-30T09:45:59.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:45:59.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:45:59.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:45:59.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:45:59.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:45:59.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:45:59.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:45:59 smithi026 ceph-mon[22362]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:46:01.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:01 smithi026 ceph-mon[27123]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:46:01.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:01 smithi026 ceph-mon[22362]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:46:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:01 smithi046 ceph-mon[23181]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:46:03.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[22362]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[27123]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:03.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:03.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:03.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:46:03.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:46:03.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:03 smithi046 ceph-mon[23181]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:46:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:46:04.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:46:05.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:04 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:05.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:04 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:04 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:06.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:05 smithi026 ceph-mon[22362]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:46:06.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:05 smithi026 ceph-mon[27123]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:46:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:05 smithi046 ceph-mon[23181]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:46:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:46:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:46:06] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-04-30T09:46:08.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:07 smithi026 ceph-mon[22362]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:08.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:07 smithi026 ceph-mon[27123]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:07 smithi046 ceph-mon[23181]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:10.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:09 smithi026 ceph-mon[22362]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:10.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:09 smithi026 ceph-mon[27123]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:10.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:09 smithi046 ceph-mon[23181]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:12.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:11 smithi026 ceph-mon[27123]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:12.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:11 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:11 smithi026 ceph-mon[22362]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:11 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:11 smithi046 ceph-mon[23181]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:11 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:14.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:13 smithi026 ceph-mon[22362]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:14.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:13 smithi026 ceph-mon[27123]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:13 smithi046 ceph-mon[23181]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:15.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:14 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:15.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:14 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:15.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:14 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:16.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:15 smithi026 ceph-mon[22362]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:16.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:15 smithi026 ceph-mon[27123]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:16.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:15 smithi046 ceph-mon[23181]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:46:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:46:16] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.33.4" 2024-04-30T09:46:18.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:17 smithi026 ceph-mon[27123]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:18.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:17 smithi026 ceph-mon[22362]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:17 smithi046 ceph-mon[23181]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:20.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:19 smithi026 ceph-mon[27123]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:20.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:19 smithi026 ceph-mon[22362]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:19 smithi046 ceph-mon[23181]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:22.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:21 smithi026 ceph-mon[27123]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:21 smithi026 ceph-mon[22362]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:21 smithi046 ceph-mon[23181]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:24.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:23 smithi026 ceph-mon[27123]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:24.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:23 smithi026 ceph-mon[22362]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:23 smithi046 ceph-mon[23181]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:24 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:24 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:25.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:24 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:26.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:25 smithi026 ceph-mon[27123]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:25 smithi026 ceph-mon[22362]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:26.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:25 smithi046 ceph-mon[23181]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:46:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:46:26] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-04-30T09:46:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:26 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:26 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:26 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:27 smithi046 ceph-mon[23181]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:28.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:27 smithi026 ceph-mon[27123]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:27 smithi026 ceph-mon[22362]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:29 smithi046 ceph-mon[23181]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:30.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:29 smithi026 ceph-mon[27123]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:30.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:29 smithi026 ceph-mon[22362]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:31 smithi026 ceph-mon[27123]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:31 smithi026 ceph-mon[22362]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:31 smithi046 ceph-mon[23181]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:34.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:33 smithi026 ceph-mon[27123]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:33 smithi026 ceph-mon[22362]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:33 smithi046 ceph-mon[23181]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:34 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:35.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:34 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:34 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:36.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:35 smithi026 ceph-mon[27123]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:36.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:35 smithi026 ceph-mon[22362]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:35 smithi046 ceph-mon[23181]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:46:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:46:36] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-04-30T09:46:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:37 smithi046 ceph-mon[23181]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:38.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:37 smithi026 ceph-mon[27123]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:37 smithi026 ceph-mon[22362]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:39 smithi046 ceph-mon[23181]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:40.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:39 smithi026 ceph-mon[27123]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:40.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:39 smithi026 ceph-mon[22362]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:41 smithi026 ceph-mon[27123]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:41 smithi026 ceph-mon[22362]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:41 smithi046 ceph-mon[23181]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:43.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:43 smithi026 ceph-mon[27123]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:43.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:43 smithi026 ceph-mon[22362]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:43 smithi046 ceph-mon[23181]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:45.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:44 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:45.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:44 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:44 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:46.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:45 smithi026 ceph-mon[22362]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:45 smithi026 ceph-mon[27123]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:45 smithi046 ceph-mon[23181]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:46.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:46:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:46:46] "GET /metrics HTTP/1.1" 200 37893 "" "Prometheus/2.33.4" 2024-04-30T09:46:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:47 smithi046 ceph-mon[23181]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:48.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:47 smithi026 ceph-mon[27123]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:48.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:47 smithi026 ceph-mon[22362]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:49 smithi046 ceph-mon[23181]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:49 smithi026 ceph-mon[27123]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:49 smithi026 ceph-mon[22362]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:52.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:51 smithi026 ceph-mon[27123]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:51 smithi026 ceph-mon[22362]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:51 smithi046 ceph-mon[23181]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:54.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:53 smithi026 ceph-mon[27123]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:54.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:53 smithi026 ceph-mon[22362]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:53 smithi046 ceph-mon[23181]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:54 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:54 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:54 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:46:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:55 smithi046 ceph-mon[23181]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:56.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:55 smithi026 ceph-mon[27123]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:56.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:55 smithi026 ceph-mon[22362]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:46:56.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:46:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:46:56] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-04-30T09:46:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:56 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:57.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:56 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:57.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:56 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:46:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:57 smithi046 ceph-mon[23181]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:58.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:57 smithi026 ceph-mon[22362]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:46:58.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:57 smithi026 ceph-mon[27123]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:46:59 smithi046 ceph-mon[23181]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:00.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:46:59 smithi026 ceph-mon[27123]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:46:59 smithi026 ceph-mon[22362]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:02.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:01 smithi026 ceph-mon[22362]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:02.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:01 smithi026 ceph-mon[27123]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:01 smithi046 ceph-mon[23181]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[27123]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[22362]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:03 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:03 smithi046 ceph-mon[23181]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:47:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:03 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:04 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:04 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:47:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:04 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:47:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:04 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:05.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:04 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:05.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:04 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:47:05.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:04 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:47:05.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:04 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:04 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:04 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:47:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:04 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:47:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:04 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:47:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:05 smithi046 ceph-mon[23181]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:06.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:05 smithi026 ceph-mon[27123]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:06.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:05 smithi026 ceph-mon[22362]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:47:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:47:06] "GET /metrics HTTP/1.1" 200 37890 "" "Prometheus/2.33.4" 2024-04-30T09:47:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:07 smithi046 ceph-mon[23181]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:08.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:07 smithi026 ceph-mon[27123]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:08.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:07 smithi026 ceph-mon[22362]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:10.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:09 smithi046 ceph-mon[23181]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:10.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:09 smithi026 ceph-mon[27123]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:10.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:09 smithi026 ceph-mon[22362]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:12.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:11 smithi026 ceph-mon[22362]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:11 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:12.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:11 smithi026 ceph-mon[27123]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:12.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:11 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:11 smithi046 ceph-mon[23181]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:11 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:14.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:13 smithi026 ceph-mon[27123]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:14.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:13 smithi026 ceph-mon[22362]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:13 smithi046 ceph-mon[23181]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:15.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:14 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:14 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:14 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:16.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:15 smithi046 ceph-mon[23181]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:16.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:15 smithi026 ceph-mon[27123]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:16.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:15 smithi026 ceph-mon[22362]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:47:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:47:16] "GET /metrics HTTP/1.1" 200 37886 "" "Prometheus/2.33.4" 2024-04-30T09:47:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:17 smithi046 ceph-mon[23181]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:17 smithi026 ceph-mon[27123]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:17 smithi026 ceph-mon[22362]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:19 smithi046 ceph-mon[23181]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:20.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:19 smithi026 ceph-mon[27123]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:19 smithi026 ceph-mon[22362]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:22.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:21 smithi026 ceph-mon[22362]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:22.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:21 smithi026 ceph-mon[27123]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:21 smithi046 ceph-mon[23181]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:24.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:23 smithi026 ceph-mon[27123]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:24.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:23 smithi026 ceph-mon[22362]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:23 smithi046 ceph-mon[23181]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:25.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:24 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:25.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:24 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:25.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:24 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:25 smithi046 ceph-mon[23181]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:26.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:25 smithi026 ceph-mon[27123]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:26.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:25 smithi026 ceph-mon[22362]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:47:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:47:26] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-04-30T09:47:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:26 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:26 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:26 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:27 smithi046 ceph-mon[23181]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:27 smithi026 ceph-mon[27123]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:27 smithi026 ceph-mon[22362]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:29 smithi046 ceph-mon[23181]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:30.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:29 smithi026 ceph-mon[27123]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:30.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:29 smithi026 ceph-mon[22362]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:31 smithi026 ceph-mon[27123]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:31 smithi026 ceph-mon[22362]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:31 smithi046 ceph-mon[23181]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:34.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:33 smithi026 ceph-mon[27123]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:33 smithi026 ceph-mon[22362]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:33 smithi046 ceph-mon[23181]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:34 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:35.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:34 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:35.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:34 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:35 smithi046 ceph-mon[23181]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:36.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:35 smithi026 ceph-mon[27123]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:36.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:35 smithi026 ceph-mon[22362]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:47:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:47:36] "GET /metrics HTTP/1.1" 200 37888 "" "Prometheus/2.33.4" 2024-04-30T09:47:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:37 smithi046 ceph-mon[23181]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:38.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:37 smithi026 ceph-mon[27123]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:37 smithi026 ceph-mon[22362]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:39 smithi046 ceph-mon[23181]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:40.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:39 smithi026 ceph-mon[22362]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:40.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:39 smithi026 ceph-mon[27123]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:42.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:41 smithi026 ceph-mon[22362]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:42.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:41 smithi026 ceph-mon[27123]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:42.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:41 smithi046 ceph-mon[23181]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:44.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:43 smithi026 ceph-mon[27123]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:43 smithi026 ceph-mon[22362]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:43 smithi046 ceph-mon[23181]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:44 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:45.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:44 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:45.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:44 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:45 smithi046 ceph-mon[23181]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:46.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:45 smithi026 ceph-mon[27123]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:46.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:45 smithi026 ceph-mon[22362]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:46.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:47:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:47:46] "GET /metrics HTTP/1.1" 200 37892 "" "Prometheus/2.33.4" 2024-04-30T09:47:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:47 smithi046 ceph-mon[23181]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:48.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:47 smithi026 ceph-mon[27123]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:48.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:47 smithi026 ceph-mon[22362]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:49 smithi046 ceph-mon[23181]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:50.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:49 smithi026 ceph-mon[27123]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:50.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:49 smithi026 ceph-mon[22362]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:52.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:51 smithi026 ceph-mon[27123]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:51 smithi026 ceph-mon[22362]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:51 smithi046 ceph-mon[23181]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:54.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:53 smithi026 ceph-mon[27123]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:54.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:53 smithi026 ceph-mon[22362]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:53 smithi046 ceph-mon[23181]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:54 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:55.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:54 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:55.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:54 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:47:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:55 smithi046 ceph-mon[23181]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:56.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:55 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:56.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:55 smithi026 ceph-mon[27123]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:56.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:55 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:56.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:55 smithi026 ceph-mon[22362]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:47:56.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:55 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:47:56.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:47:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:47:56] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-04-30T09:47:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:57 smithi046 ceph-mon[23181]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:58.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:57 smithi026 ceph-mon[27123]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:47:58.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:57 smithi026 ceph-mon[22362]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:47:59 smithi046 ceph-mon[23181]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:00.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:47:59 smithi026 ceph-mon[27123]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:47:59 smithi026 ceph-mon[22362]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:02.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:01 smithi026 ceph-mon[22362]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:02.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:01 smithi026 ceph-mon[27123]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:01 smithi046 ceph-mon[23181]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:03 smithi026 ceph-mon[27123]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:03 smithi026 ceph-mon[22362]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:03 smithi046 ceph-mon[23181]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:04 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:04 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:48:05.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:04 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:05.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:04 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:48:05.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:04 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:05.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:04 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:48:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:05 smithi046 ceph-mon[23181]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:05 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:48:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:05 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:48:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:05 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:48:06.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:05 smithi026 ceph-mon[22362]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:06.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:05 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:48:06.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:05 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:48:06.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:05 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:48:06.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:05 smithi026 ceph-mon[27123]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:06.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:05 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:48:06.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:05 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:48:06.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:05 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' 2024-04-30T09:48:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:48:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:48:06] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-04-30T09:48:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:07 smithi046 ceph-mon[23181]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:08.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:07 smithi026 ceph-mon[27123]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:08.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:07 smithi026 ceph-mon[22362]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:10.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:09 smithi046 ceph-mon[23181]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:10.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:09 smithi026 ceph-mon[27123]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:10.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:09 smithi026 ceph-mon[22362]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:12.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:11 smithi026 ceph-mon[27123]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:12.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:11 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:11 smithi026 ceph-mon[22362]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:11 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:11 smithi046 ceph-mon[23181]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:11 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:14.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:13 smithi026 ceph-mon[27123]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:14.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:13 smithi026 ceph-mon[22362]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:13 smithi046 ceph-mon[23181]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:15.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:14 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:14 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:15.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:14 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:16.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:15 smithi046 ceph-mon[23181]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:16.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:15 smithi026 ceph-mon[27123]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:16.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:15 smithi026 ceph-mon[22362]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:48:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:48:16] "GET /metrics HTTP/1.1" 200 37899 "" "Prometheus/2.33.4" 2024-04-30T09:48:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:17 smithi046 ceph-mon[23181]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:17 smithi026 ceph-mon[27123]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:17 smithi026 ceph-mon[22362]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:19 smithi046 ceph-mon[23181]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:20.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:19 smithi026 ceph-mon[27123]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:20.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:19 smithi026 ceph-mon[22362]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:22.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:21 smithi026 ceph-mon[27123]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:21 smithi026 ceph-mon[22362]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:21 smithi046 ceph-mon[23181]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:24.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:23 smithi026 ceph-mon[27123]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:24.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:23 smithi026 ceph-mon[22362]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:23 smithi046 ceph-mon[23181]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:25.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:24 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:25.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:24 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:25.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:24 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:25 smithi046 ceph-mon[23181]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:25 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:26.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:25 smithi026 ceph-mon[27123]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:26.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:25 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:26.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:25 smithi026 ceph-mon[22362]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:26.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:25 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:48:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:48:26] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-04-30T09:48:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:27 smithi046 ceph-mon[23181]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:28.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:27 smithi026 ceph-mon[27123]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:27 smithi026 ceph-mon[22362]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:30.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:29 smithi026 ceph-mon[27123]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:30.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:29 smithi026 ceph-mon[22362]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:30.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:29 smithi046 ceph-mon[23181]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:32.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:31 smithi026 ceph-mon[22362]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:31 smithi026 ceph-mon[27123]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:31 smithi046 ceph-mon[23181]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:34.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:33 smithi026 ceph-mon[27123]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:33 smithi026 ceph-mon[22362]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:33 smithi046 ceph-mon[23181]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:34 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:35.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:34 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:35.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:34 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:35 smithi046 ceph-mon[23181]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:36.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:35 smithi026 ceph-mon[22362]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:36.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:35 smithi026 ceph-mon[27123]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:48:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:48:36] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-04-30T09:48:38.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:37 smithi026 ceph-mon[27123]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:37 smithi026 ceph-mon[22362]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:37 smithi046 ceph-mon[23181]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:40.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:39 smithi026 ceph-mon[27123]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:48:40.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:39 smithi026 ceph-mon[22362]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:48:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:39 smithi046 ceph-mon[23181]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:48:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:41 smithi026 ceph-mon[27123]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:48:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:41 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:41 smithi026 ceph-mon[22362]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:48:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:41 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:41 smithi046 ceph-mon[23181]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:48:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:41 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:43.733 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T09:48:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:43 smithi046 ceph-mon[23181]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:44.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:43 smithi026 ceph-mon[27123]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:44.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:43 smithi026 ceph-mon[22362]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:45.166 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:44 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:45.167 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:44 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:44 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:48:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (9m) 2m ago 17m 23.3M - 0.23.0 ba2b418f427c 1648d8f22b65 2024-04-30T09:48:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (8m) 2m ago 16m 51.5M - 8.3.5 dad864ee21e9 9bd49a225988 2024-04-30T09:48:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 running (2m) 2m ago 16m 50.4M - 3.5 e1d6a67b021e 5bb76058e663 2024-04-30T09:48:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283 running (11m) 2m ago 21m 449M - 19.0.0-3357-g3a873b0c c50cf5fa633a 53731c91d116 2024-04-30T09:48:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (6m) 2m ago 23m 496M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:48:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (23m) 2m ago 23m 66.1M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:48:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (22m) 2m ago 22m 48.4M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (22m) 2m ago 22m 50.6M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (9m) 2m ago 17m 17.7M - 1.3.1 1dbe0e931976 b6af5622fe4c 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (9m) 2m ago 17m 16.9M - 1.3.1 1dbe0e931976 23b6576e6bb5 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (21m) 2m ago 21m 52.8M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (21m) 2m ago 21m 52.2M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (20m) 2m ago 20m 52.2M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (20m) 2m ago 20m 50.5M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (19m) 2m ago 19m 54.8M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (19m) 2m ago 19m 51.5M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (19m) 2m ago 19m 50.9M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (18m) 2m ago 18m 52.4M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (2m) 2m ago 18m 44.7M - 2.33.4 514e6a882f6e a6e8333ff844 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (16m) 2m ago 16m 73.7M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:48:45.650 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (16m) 2m ago 16m 73.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:48:46.250 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:45 smithi026 ceph-mon[27123]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:48:46.250 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:45 smithi026 ceph-mon[22362]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:48:46.252 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions' 2024-04-30T09:48:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:45 smithi046 ceph-mon[23181]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:48:46.506 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:48:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:48:46] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-04-30T09:48:47.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:46 smithi026 ceph-mon[22362]: from='client.25198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:48:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:46 smithi026 ceph-mon[27123]: from='client.25198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:48:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:46 smithi046 ceph-mon[23181]: from='client.25198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:48:48.197 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:48:48.197 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:48:48.197 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:48:48.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:48:48.197 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:48:48.198 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:48:48.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:47 smithi026 ceph-mon[27123]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:47 smithi026 ceph-mon[22362]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:47 smithi046 ceph-mon[23181]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:48.914 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph -s' 2024-04-30T09:48:49.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:48 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1454445643' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:48:49.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:48 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1454445643' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:48:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:48 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1454445643' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:48:50.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:49 smithi026 ceph-mon[27123]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:50.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:49 smithi026 ceph-mon[22362]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:49 smithi046 ceph-mon[23181]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: cluster: 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: id: 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: health: HEALTH_OK 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: services: 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: mon: 3 daemons, quorum a,c,b (age 22m) 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: mgr: y(active, since 3m), standbys: x 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: osd: 8 osds: 8 up (since 18m), 8 in (since 19m) 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: rgw: 2 daemons active (2 hosts, 1 zones) 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:48:51.036 INFO:teuthology.orchestra.run.smithi026.stdout: data: 2024-04-30T09:48:51.037 INFO:teuthology.orchestra.run.smithi026.stdout: pools: 6 pools, 161 pgs 2024-04-30T09:48:51.037 INFO:teuthology.orchestra.run.smithi026.stdout: objects: 209 objects, 585 KiB 2024-04-30T09:48:51.037 INFO:teuthology.orchestra.run.smithi026.stdout: usage: 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:48:51.037 INFO:teuthology.orchestra.run.smithi026.stdout: pgs: 161 active+clean 2024-04-30T09:48:51.037 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:48:51.037 INFO:teuthology.orchestra.run.smithi026.stdout: io: 2024-04-30T09:48:51.037 INFO:teuthology.orchestra.run.smithi026.stdout: client: 853 B/s rd, 0 op/s rd, 0 op/s wr 2024-04-30T09:48:51.037 INFO:teuthology.orchestra.run.smithi026.stdout: 2024-04-30T09:48:51.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T09:48:51.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:51 smithi026 ceph-mon[27123]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:51.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:51 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4038516037' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:48:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:51 smithi026 ceph-mon[22362]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:51 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4038516037' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:48:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:51 smithi046 ceph-mon[23181]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:51 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4038516037' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-30T09:48:53.756 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_OK 2024-04-30T09:48:54.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:53 smithi026 ceph-mon[27123]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:54.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:53 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/555297932' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:48:54.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:53 smithi026 ceph-mon[22362]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:54.071 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:53 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/555297932' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:48:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:53 smithi046 ceph-mon[23181]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:53 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/555297932' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:48:54.591 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-30T09:48:55.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:54 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:55.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:54 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:54 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:48:56.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:55 smithi026 ceph-mon[22362]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:55 smithi026 ceph-mon[22362]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:55 smithi026 ceph-mon[27123]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:55 smithi026 ceph-mon[27123]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:56.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:55 smithi046 ceph-mon[23181]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:48:56.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:55 smithi046 ceph-mon[23181]: from='mgr.15210 172.21.15.26:0/1952920890' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:48:56.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:48:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:48:56] "GET /metrics HTTP/1.1" 200 37901 "" "Prometheus/2.33.4" 2024-04-30T09:48:56.721 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:48:57.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph mgr fail' 2024-04-30T09:48:57.265 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:56 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4070266250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:48:57.266 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:56 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4070266250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:48:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:56 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4070266250' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:48:58.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:48:57 smithi026 ceph-mon[27123]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:58.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:48:57 smithi026 ceph-mon[22362]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:48:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:48:57 smithi046 ceph-mon[23181]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:00.334 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-mon[27123]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:00.334 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3008674937' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:49:00.334 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:49:00.334 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-mon[27123]: osdmap e105: 8 total, 8 up, 8 in 2024-04-30T09:49:00.335 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T09:49:00.000+0000 7fc36a925700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-30T09:49:00.335 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ignoring --setuser ceph since I am not root 2024-04-30T09:49:00.335 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ignoring --setgroup ceph since I am not root 2024-04-30T09:49:00.335 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-mgr[60498]: -- 172.21.15.26:0/1889323971 <== mon.1 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562d1e5e05a0 con 0x562d1e5b4c00 2024-04-30T09:49:00.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-mon[22362]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:00.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3008674937' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:49:00.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:49:00.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:00 smithi026 ceph-mon[22362]: osdmap e105: 8 total, 8 up, 8 in 2024-04-30T09:49:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-mon[23181]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3008674937' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:49:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd=[{"prefix": "mgr fail"}]: dispatch 2024-04-30T09:49:00.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-mon[23181]: osdmap e105: 8 total, 8 up, 8 in 2024-04-30T09:49:00.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:00] ENGINE Bus STOPPING 2024-04-30T09:49:00.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:49:00.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:00] ENGINE Bus STOPPED 2024-04-30T09:49:00.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:00] ENGINE Bus STARTING 2024-04-30T09:49:00.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:00] ENGINE Serving on http://:::9283 2024-04-30T09:49:00.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:00] ENGINE Bus STARTED 2024-04-30T09:49:00.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'sleep 180' 2024-04-30T09:49:01.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-30T09:49:01.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: mgrmap e33: x(active, starting, since 0.836172s) 2024-04-30T09:49:01.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:49:01.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:49:01.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:49:01.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:49:01.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:49:01.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: Manager daemon x is now available 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:49:01.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: mgrmap e33: x(active, starting, since 0.836172s) 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:49:01.257 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: Manager daemon x is now available 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:49:01.258 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:01 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:49:01.482 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='client.? ' entity='client.admin' cmd='[{"prefix": "mgr fail"}]': finished 2024-04-30T09:49:01.482 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: mgrmap e33: x(active, starting, since 0.836172s) 2024-04-30T09:49:01.482 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:49:01.482 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:49:01.482 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:49:01.482 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:49:01.482 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:49:01.482 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:49:01.482 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: Manager daemon x is now available 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:49:01.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:01 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-30T09:49:02.162 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[27123]: mgrmap e34: x(active, since 1.85003s) 2024-04-30T09:49:02.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[27123]: [30/Apr/2024:09:49:01] ENGINE Bus STARTING 2024-04-30T09:49:02.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[27123]: [30/Apr/2024:09:49:01] ENGINE Serving on https://172.21.15.46:7150 2024-04-30T09:49:02.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[27123]: [30/Apr/2024:09:49:01] ENGINE Serving on http://172.21.15.46:8765 2024-04-30T09:49:02.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[27123]: [30/Apr/2024:09:49:01] ENGINE Bus STARTED 2024-04-30T09:49:02.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[22362]: mgrmap e34: x(active, since 1.85003s) 2024-04-30T09:49:02.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[22362]: [30/Apr/2024:09:49:01] ENGINE Bus STARTING 2024-04-30T09:49:02.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[22362]: [30/Apr/2024:09:49:01] ENGINE Serving on https://172.21.15.46:7150 2024-04-30T09:49:02.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[22362]: [30/Apr/2024:09:49:01] ENGINE Serving on http://172.21.15.46:8765 2024-04-30T09:49:02.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:02 smithi026 ceph-mon[22362]: [30/Apr/2024:09:49:01] ENGINE Bus STARTED 2024-04-30T09:49:02.311 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:02 smithi046 ceph-mon[23181]: mgrmap e34: x(active, since 1.85003s) 2024-04-30T09:49:02.311 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:02 smithi046 ceph-mon[23181]: [30/Apr/2024:09:49:01] ENGINE Bus STARTING 2024-04-30T09:49:02.311 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:02 smithi046 ceph-mon[23181]: [30/Apr/2024:09:49:01] ENGINE Serving on https://172.21.15.46:7150 2024-04-30T09:49:02.311 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:02 smithi046 ceph-mon[23181]: [30/Apr/2024:09:49:01] ENGINE Serving on http://172.21.15.46:8765 2024-04-30T09:49:02.311 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:02 smithi046 ceph-mon[23181]: [30/Apr/2024:09:49:01] ENGINE Bus STARTED 2024-04-30T09:49:03.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:03 smithi026 ceph-mon[27123]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:03.301 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:03 smithi026 ceph-mon[22362]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:03 smithi046 ceph-mon[23181]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:04 smithi026 ceph-mon[27123]: mgrmap e35: x(active, since 3s) 2024-04-30T09:49:04.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:04 smithi026 ceph-mon[27123]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:04 smithi026 ceph-mon[22362]: mgrmap e35: x(active, since 3s) 2024-04-30T09:49:04.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:04 smithi026 ceph-mon[22362]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:04.412 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:04 smithi046 ceph-mon[23181]: mgrmap e35: x(active, since 3s) 2024-04-30T09:49:04.412 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:04 smithi046 ceph-mon[23181]: from='client.25186 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:05.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:05 smithi026 ceph-mon[27123]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:05.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:05 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:05.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:05 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:05.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:05 smithi026 ceph-mon[22362]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:05.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:05 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:05.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:05 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:05 smithi046 ceph-mon[23181]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:05 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:05 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[27123]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:07.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.452 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[22362]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:07.452 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.452 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.452 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.453 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:07 smithi046 ceph-mon[23181]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:07 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:07 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:07 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:07 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:07.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:49:07] ENGINE Bus STARTING 2024-04-30T09:49:07.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: CherryPy Checker: 2024-04-30T09:49:07.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: The Application mounted at '' has an empty config. 2024-04-30T09:49:07.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T09:49:07.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:49:07] ENGINE Serving on http://:::9283 2024-04-30T09:49:07.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:49:07 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:49:07] ENGINE Bus STARTED 2024-04-30T09:49:08.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: Standby manager daemon y started 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:49:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[22362]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: Standby manager daemon y started 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:49:08.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:08 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:49:08.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:08.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:08.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: Standby manager daemon y started 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:49:08.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:08 smithi046 ceph-mon[23181]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:49:09.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:49:09.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:49:09.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: mgrmap e36: x(active, since 9s), standbys: y 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:09.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[27123]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:49:09.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:49:09.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:49:09.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:49:09.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:49:09.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:49:09.530 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: mgrmap e36: x(active, since 9s), standbys: y 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:09.531 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:09 smithi026 ceph-mon[22362]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:49:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:49:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:49:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:49:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:49:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: mgrmap e36: x(active, since 9s), standbys: y 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:09 smithi046 ceph-mon[23181]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:49:11.557+0000 7f31c552d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:11.862 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:49:11.863 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:11.863 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:49:11.863 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:11.863 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:11.863 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:11.863 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:11.863 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:11.863 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:11.863 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return future.result(timeout) 2024-04-30T09:49:11.864 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.__get_result() 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: raise self._exception 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: use_current_daemon_image=reconfig, 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:11.865 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:11.866 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:11.867 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:11.867 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:12.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:11 smithi026 ceph-mon[22362]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:49:12.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:11 smithi026 ceph-mon[27123]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:49:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:11 smithi046 ceph-mon[23181]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:49:12.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:12 smithi046 ceph-mon[23181]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:12.985 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:12.985 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:12.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: return future.result(timeout) 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:49:12.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.__get_result() 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: raise self._exception 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:12.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:12 smithi046 ceph-mon[23181]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:49:12.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:12 smithi046 ceph-mon[23181]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:12 smithi026 ceph-mon[22362]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:49:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:13.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:13.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:12 smithi026 ceph-mon[27123]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:13.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:13.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:13.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:13.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:13.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:13.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:13.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:13.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:49:13.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:13.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:13.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:12 smithi026 ceph-mon[22362]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:12 smithi026 ceph-mon[22362]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:13.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:13.206 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:13.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:12 smithi026 ceph-mon[27123]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:49:13.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:12 smithi026 ceph-mon[27123]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:49:14.080 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:13 smithi046 ceph-mon[23181]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:49:14.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:13 smithi026 ceph-mon[22362]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:49:14.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:13 smithi026 ceph-mon[27123]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:49:15.004 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:14 smithi046 systemd[1]: Stopping Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:49:15.005 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:14 smithi046 bash[55094]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:49:15.294 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.003Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-30T09:49:15.294 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.003Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-30T09:49:15.294 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.003Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-30T09:49:15.294 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.003Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-30T09:49:15.294 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.003Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-30T09:49:15.294 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.003Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-30T09:49:15.294 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.004Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-30T09:49:15.295 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.004Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-30T09:49:15.295 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.004Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-30T09:49:15.295 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.004Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-30T09:49:15.295 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.004Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-30T09:49:15.295 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[52188]: ts=2024-04-30T09:49:15.004Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-30T09:49:15.295 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 podman[55106]: 2024-04-30 09:49:15.014058888 +0000 UTC m=+0.110032724 container died a6e8333ff844491e379c59aa7eb433ff439815dfb82560502eedacec52b24346 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:49:15.655 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 podman[55106]: 2024-04-30 09:49:15.420213035 +0000 UTC m=+0.516186862 container cleanup a6e8333ff844491e379c59aa7eb433ff439815dfb82560502eedacec52b24346 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:49:15.655 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 bash[55106]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a 2024-04-30T09:49:15.655 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 bash[55134]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:49:15.914 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-mon[23181]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:49:15.915 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:15 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:15.915 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 podman[55120]: 2024-04-30 09:49:15.654729641 +0000 UTC m=+0.643230382 container remove a6e8333ff844491e379c59aa7eb433ff439815dfb82560502eedacec52b24346 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:49:15.915 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-30T09:49:15.915 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 systemd[1]: Stopped Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:49:15.915 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.415s CPU time. 2024-04-30T09:49:15.915 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 systemd[1]: Starting Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:49:15.915 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 podman[55203]: 2024-04-30 09:49:15.790215498 +0000 UTC m=+0.019915031 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-30T09:49:15.915 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:15 smithi046 podman[55203]: 2024-04-30 09:49:15.914309424 +0000 UTC m=+0.144008937 container create a48285f81913d73589aa09804e26d7ca63e646c97b554b9a7fb06cb10b1c13b0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:49:16.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:15 smithi026 ceph-mon[27123]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:49:16.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:15 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:16.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:15 smithi026 ceph-mon[22362]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:49:16.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:15 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:16.209 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 podman[55203]: 2024-04-30 09:49:16.118008527 +0000 UTC m=+0.347708041 container init a48285f81913d73589aa09804e26d7ca63e646c97b554b9a7fb06cb10b1c13b0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:49:16.209 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 podman[55203]: 2024-04-30 09:49:16.120701697 +0000 UTC m=+0.350401214 container start a48285f81913d73589aa09804e26d7ca63e646c97b554b9a7fb06cb10b1c13b0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:49:16.209 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.144Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-30T09:49:16.209 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.144Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-30T09:49:16.209 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.144Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-30T09:49:16.209 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.144Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi046 (none))" 2024-04-30T09:49:16.209 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.144Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-30T09:49:16.209 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.144Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-30T09:49:16.210 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.146Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-30T09:49:16.210 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.146Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-30T09:49:16.210 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.147Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-30T09:49:16.210 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.148Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-30T09:49:16.210 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.148Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.488µs 2024-04-30T09:49:16.210 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.148Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-30T09:49:16.210 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 bash[55203]: a48285f81913d73589aa09804e26d7ca63e646c97b554b9a7fb06cb10b1c13b0 2024-04-30T09:49:16.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 systemd[1]: Started Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:49:16.484 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.335Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-04-30T09:49:16.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:16] ENGINE Bus STOPPING 2024-04-30T09:49:16.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:49:16.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:16] ENGINE Bus STOPPED 2024-04-30T09:49:16.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:16] ENGINE Bus STARTING 2024-04-30T09:49:16.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:16] ENGINE Serving on http://:::9283 2024-04-30T09:49:16.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:49:16] ENGINE Bus STARTED 2024-04-30T09:49:16.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.570Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-04-30T09:49:16.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.784Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-04-30T09:49:16.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.788Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-04-30T09:49:16.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.788Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-04-30T09:49:16.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.788Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.312µs wal_replay_duration=639.391268ms total_replay_duration=639.441638ms 2024-04-30T09:49:16.824 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.794Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-30T09:49:16.825 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.794Z caller=main.go:947 level=info msg="TSDB started" 2024-04-30T09:49:16.825 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.794Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-30T09:49:16.825 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.823Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=29.301301ms db_storage=944ns remote_storage=1.749µs web_handler=1.059µs query_engine=873ns scrape=661.793µs scrape_sd=177.885µs notify=19.602µs notify_sd=14.94µs rules=27.887545ms 2024-04-30T09:49:16.825 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:49:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:49:16.823Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-30T09:49:17.381 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:17 smithi046 ceph-mon[23181]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:49:17.381 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:17 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:17.381 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:17 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:17.381 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:17 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:49:17.381 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:17 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:49:17.381 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:17 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:49:17.381 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:17 smithi046 ceph-mon[23181]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:49:17.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:17 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:17.382 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:17 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[22362]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[22362]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:49:17.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[27123]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:49:17.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:17.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:17.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:49:17.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:49:17.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:49:17.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[27123]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046:9095"}]: dispatch 2024-04-30T09:49:17.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:17.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:17 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:49:19.547 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:19 smithi046 ceph-mon[23181]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:49:19.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:19 smithi026 ceph-mon[27123]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:49:19.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:19 smithi026 ceph-mon[22362]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:21.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[27123]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:21.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:21 smithi026 ceph-mon[22362]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:49:21.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:49:21.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:21.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:21.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:21.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:49:21.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:49:21.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:49:21.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:21.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:49:21.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:49:21.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:21 smithi046 ceph-mon[23181]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:49:23.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:23 smithi026 ceph-mon[22362]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:49:23.609 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:23 smithi026 ceph-mon[27123]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:49:23.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:23 smithi046 ceph-mon[23181]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 426 B/s rd, 0 op/s 2024-04-30T09:49:24.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:49:24.008+0000 7f31c552d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:24.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:24.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:24.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:24.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:24.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:49:24.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:24.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:24.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:49:24.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return future.result(timeout) 2024-04-30T09:49:24.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:49:24.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.__get_result() 2024-04-30T09:49:24.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:49:24.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: raise self._exception 2024-04-30T09:49:24.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:49:24.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: use_current_daemon_image=reconfig, 2024-04-30T09:49:24.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:49:24.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:24.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:24.490 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:24.491 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:24.491 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:24.491 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:24.491 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:24.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:24 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2369361050' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:49:24.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:24 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1721572247' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1357068648"}]: dispatch 2024-04-30T09:49:24.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:24 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1357068648"}]: dispatch 2024-04-30T09:49:24.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:24 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2369361050' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:49:24.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:24 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1721572247' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1357068648"}]: dispatch 2024-04-30T09:49:24.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:24 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1357068648"}]: dispatch 2024-04-30T09:49:24.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2369361050' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T09:49:24.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1721572247' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1357068648"}]: dispatch 2024-04-30T09:49:24.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:24 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1357068648"}]: dispatch 2024-04-30T09:49:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:49:25] "GET /metrics HTTP/1.1" 200 37895 "" "Prometheus/2.33.4" 2024-04-30T09:49:25.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[27123]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:25.948 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-30T09:49:25.949 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.950 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.951 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:25.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[22362]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.951 INFO:journalctl@ceph.mon.a.smithi026.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.951 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.951 INFO:journalctl@ceph.mon.a.smithi026.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.951 INFO:journalctl@ceph.mon.a.smithi026.stdout: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:25.951 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:25.951 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:25.951 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:25.952 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:49:25.953 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.954 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.955 INFO:journalctl@ceph.mon.a.smithi026.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.955 INFO:journalctl@ceph.mon.a.smithi026.stdout: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:25.955 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:25.955 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:25.955 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:25.955 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.957 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.957 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.957 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:49:25.957 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[22362]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:25.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1357068648"}]': finished 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[22362]: osdmap e106: 8 total, 8 up, 8 in 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3138767692' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3996716730"}]: dispatch 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:25.959 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:25.960 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:25.960 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:25.960 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.960 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[27123]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:25.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1357068648"}]': finished 2024-04-30T09:49:25.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[27123]: osdmap e106: 8 total, 8 up, 8 in 2024-04-30T09:49:25.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:25 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3138767692' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3996716730"}]: dispatch 2024-04-30T09:49:25.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:25 smithi046 ceph-mon[23181]: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.986 INFO:journalctl@ceph.mon.b.smithi046.stdout: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:25.987 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:49:25.988 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: return future.result(timeout) 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.__get_result() 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: raise self._exception 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi-foo-smithi026-romfrr 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 125 from /usr/bin/podman container inspect --format {{.State.Status}} ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: /usr/bin/podman: stderr Error: no such container ceph-703f4aea-06d3-11ef-bc93-c7b262605968-iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.989 INFO:journalctl@ceph.mon.b.smithi046.stdout: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:49:25.990 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:25 smithi046 ceph-mon[23181]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:25 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1357068648"}]': finished 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:25 smithi046 ceph-mon[23181]: osdmap e106: 8 total, 8 up, 8 in 2024-04-30T09:49:25.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:25 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3138767692' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3996716730"}]: dispatch 2024-04-30T09:49:26.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:26 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3138767692' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3996716730"}]': finished 2024-04-30T09:49:26.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:26 smithi026 ceph-mon[27123]: osdmap e107: 8 total, 8 up, 8 in 2024-04-30T09:49:26.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:26 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2772793090' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1133416977"}]: dispatch 2024-04-30T09:49:26.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:26 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3138767692' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3996716730"}]': finished 2024-04-30T09:49:26.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:26 smithi026 ceph-mon[22362]: osdmap e107: 8 total, 8 up, 8 in 2024-04-30T09:49:26.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:26 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2772793090' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1133416977"}]: dispatch 2024-04-30T09:49:26.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:26 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3138767692' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3996716730"}]': finished 2024-04-30T09:49:26.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:26 smithi046 ceph-mon[23181]: osdmap e107: 8 total, 8 up, 8 in 2024-04-30T09:49:26.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:26 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2772793090' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1133416977"}]: dispatch 2024-04-30T09:49:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[27123]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:28.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2772793090' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1133416977"}]': finished 2024-04-30T09:49:28.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[27123]: osdmap e108: 8 total, 8 up, 8 in 2024-04-30T09:49:28.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2102104819' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2223015638"}]: dispatch 2024-04-30T09:49:28.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2223015638"}]: dispatch 2024-04-30T09:49:28.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[22362]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:28.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2772793090' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1133416977"}]': finished 2024-04-30T09:49:28.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[22362]: osdmap e108: 8 total, 8 up, 8 in 2024-04-30T09:49:28.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2102104819' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2223015638"}]: dispatch 2024-04-30T09:49:28.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:27 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2223015638"}]: dispatch 2024-04-30T09:49:28.586 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:27 smithi046 ceph-mon[23181]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:49:28.586 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:27 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2772793090' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1133416977"}]': finished 2024-04-30T09:49:28.586 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:27 smithi046 ceph-mon[23181]: osdmap e108: 8 total, 8 up, 8 in 2024-04-30T09:49:28.586 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:27 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2102104819' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2223015638"}]: dispatch 2024-04-30T09:49:28.586 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:27 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2223015638"}]: dispatch 2024-04-30T09:49:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:28 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2223015638"}]': finished 2024-04-30T09:49:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:28 smithi026 ceph-mon[27123]: osdmap e109: 8 total, 8 up, 8 in 2024-04-30T09:49:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:28 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/78025646' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3076910185"}]: dispatch 2024-04-30T09:49:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:28 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3076910185"}]: dispatch 2024-04-30T09:49:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:28 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2223015638"}]': finished 2024-04-30T09:49:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:28 smithi026 ceph-mon[22362]: osdmap e109: 8 total, 8 up, 8 in 2024-04-30T09:49:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:28 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/78025646' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3076910185"}]: dispatch 2024-04-30T09:49:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:28 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3076910185"}]: dispatch 2024-04-30T09:49:29.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:28 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/2223015638"}]': finished 2024-04-30T09:49:29.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:28 smithi046 ceph-mon[23181]: osdmap e109: 8 total, 8 up, 8 in 2024-04-30T09:49:29.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:28 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/78025646' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3076910185"}]: dispatch 2024-04-30T09:49:29.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:28 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3076910185"}]: dispatch 2024-04-30T09:49:30.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[27123]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:49:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3076910185"}]': finished 2024-04-30T09:49:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[27123]: osdmap e110: 8 total, 8 up, 8 in 2024-04-30T09:49:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3191172177' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2223015638"}]: dispatch 2024-04-30T09:49:30.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2223015638"}]: dispatch 2024-04-30T09:49:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[22362]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:49:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3076910185"}]': finished 2024-04-30T09:49:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[22362]: osdmap e110: 8 total, 8 up, 8 in 2024-04-30T09:49:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3191172177' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2223015638"}]: dispatch 2024-04-30T09:49:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:29 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2223015638"}]: dispatch 2024-04-30T09:49:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:29 smithi046 ceph-mon[23181]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:49:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:29 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/3076910185"}]': finished 2024-04-30T09:49:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:29 smithi046 ceph-mon[23181]: osdmap e110: 8 total, 8 up, 8 in 2024-04-30T09:49:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:29 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3191172177' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2223015638"}]: dispatch 2024-04-30T09:49:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:29 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2223015638"}]: dispatch 2024-04-30T09:49:31.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:30 smithi026 ceph-mon[27123]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2223015638"}]': finished 2024-04-30T09:49:31.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:30 smithi026 ceph-mon[27123]: osdmap e111: 8 total, 8 up, 8 in 2024-04-30T09:49:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:30 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:30 smithi026 ceph-mon[22362]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2223015638"}]': finished 2024-04-30T09:49:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:30 smithi026 ceph-mon[22362]: osdmap e111: 8 total, 8 up, 8 in 2024-04-30T09:49:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:30 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:30 smithi046 ceph-mon[23181]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/2223015638"}]': finished 2024-04-30T09:49:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:30 smithi046 ceph-mon[23181]: osdmap e111: 8 total, 8 up, 8 in 2024-04-30T09:49:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:30 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:31 smithi026 ceph-mon[27123]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:49:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:31 smithi026 ceph-mon[22362]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:49:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:31 smithi046 ceph-mon[23181]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 95 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:49:34.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:33 smithi026 ceph-mon[27123]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-04-30T09:49:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:33 smithi026 ceph-mon[22362]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-04-30T09:49:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:33 smithi046 ceph-mon[23181]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 994 B/s rd, 0 op/s 2024-04-30T09:49:35.172 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:34 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:34 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:34 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:35.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:49:35] "GET /metrics HTTP/1.1" 200 37900 "" "Prometheus/2.33.4" 2024-04-30T09:49:36.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:35 smithi026 ceph-mon[22362]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-04-30T09:49:36.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:35 smithi026 ceph-mon[27123]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-04-30T09:49:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:35 smithi046 ceph-mon[23181]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-04-30T09:49:38.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:37 smithi026 ceph-mon[22362]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-30T09:49:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:37 smithi026 ceph-mon[27123]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-30T09:49:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:37 smithi046 ceph-mon[23181]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s 2024-04-30T09:49:40.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:39 smithi026 ceph-mon[27123]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:49:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:39 smithi026 ceph-mon[22362]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:49:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:39 smithi046 ceph-mon[23181]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:49:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:41 smithi026 ceph-mon[27123]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1010 B/s rd, 0 op/s 2024-04-30T09:49:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:41 smithi026 ceph-mon[22362]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1010 B/s rd, 0 op/s 2024-04-30T09:49:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:41 smithi046 ceph-mon[23181]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1010 B/s rd, 0 op/s 2024-04-30T09:49:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:43 smithi026 ceph-mon[27123]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:43 smithi026 ceph-mon[22362]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:43 smithi046 ceph-mon[23181]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:45.171 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:44 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:45.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:44 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:45.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:44 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:45.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:49:45] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-04-30T09:49:46.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:45 smithi026 ceph-mon[27123]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:45 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:45 smithi026 ceph-mon[22362]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:45 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:45 smithi046 ceph-mon[23181]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:45 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:49:48.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:47 smithi026 ceph-mon[27123]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:47 smithi026 ceph-mon[22362]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:47 smithi046 ceph-mon[23181]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:50.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:49 smithi026 ceph-mon[27123]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:49 smithi026 ceph-mon[22362]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:49 smithi046 ceph-mon[23181]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:52.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:51 smithi026 ceph-mon[27123]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:51 smithi026 ceph-mon[22362]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:51 smithi046 ceph-mon[23181]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:54.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:53 smithi026 ceph-mon[27123]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:54.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:53 smithi026 ceph-mon[22362]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:53 smithi046 ceph-mon[23181]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:49:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:54 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:54 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:54 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:49:55.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:49:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:49:55] "GET /metrics HTTP/1.1" 200 37894 "" "Prometheus/2.33.4" 2024-04-30T09:49:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:55 smithi026 ceph-mon[27123]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:55 smithi026 ceph-mon[22362]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:55 smithi046 ceph-mon[23181]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:58.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:57 smithi026 ceph-mon[27123]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:58.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:57 smithi026 ceph-mon[22362]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:49:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:57 smithi046 ceph-mon[23181]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:00.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:49:59 smithi026 ceph-mon[27123]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:00.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:49:59 smithi026 ceph-mon[22362]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:49:59 smithi046 ceph-mon[23181]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:01.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:00 smithi026 ceph-mon[22362]: overall HEALTH_OK 2024-04-30T09:50:01.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:00 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:01.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:00 smithi026 ceph-mon[27123]: overall HEALTH_OK 2024-04-30T09:50:01.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:00 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:00 smithi046 ceph-mon[23181]: overall HEALTH_OK 2024-04-30T09:50:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:00 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:02.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:01 smithi026 ceph-mon[22362]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:02.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:01 smithi026 ceph-mon[27123]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:01 smithi046 ceph-mon[23181]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:03 smithi046 ceph-mon[23181]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:04.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:03 smithi026 ceph-mon[27123]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:03 smithi026 ceph-mon[22362]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:04 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:05.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:50:05] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-04-30T09:50:05.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:04 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:04 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:05 smithi046 ceph-mon[23181]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:06.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:05 smithi026 ceph-mon[27123]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:06.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:05 smithi026 ceph-mon[22362]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:07 smithi046 ceph-mon[23181]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:08.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:07 smithi026 ceph-mon[22362]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:08.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:07 smithi026 ceph-mon[27123]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:10.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:09 smithi046 ceph-mon[23181]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:10.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:09 smithi026 ceph-mon[27123]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:10.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:09 smithi026 ceph-mon[22362]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:11 smithi046 ceph-mon[23181]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:12.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:11 smithi026 ceph-mon[27123]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:11 smithi026 ceph-mon[22362]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:13 smithi026 ceph-mon[27123]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:13.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:13 smithi026 ceph-mon[22362]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:13 smithi046 ceph-mon[23181]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:14.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:14 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:14 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:14 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:15.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:15 smithi026 ceph-mon[27123]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:15 smithi026 ceph-mon[22362]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:15 smithi046 ceph-mon[23181]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:50:15] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-04-30T09:50:16.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:16 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:16 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:16 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:17.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:17 smithi026 ceph-mon[27123]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:17 smithi026 ceph-mon[22362]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:17 smithi046 ceph-mon[23181]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:19.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:19 smithi026 ceph-mon[27123]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:19 smithi026 ceph-mon[22362]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:19 smithi046 ceph-mon[23181]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:22.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:21 smithi026 ceph-mon[27123]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:21 smithi026 ceph-mon[22362]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:21 smithi046 ceph-mon[23181]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:24.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:23 smithi026 ceph-mon[22362]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:24.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:23 smithi026 ceph-mon[27123]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:23 smithi046 ceph-mon[23181]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:25.172 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:24 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:25.173 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:24 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:50:25.173 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:24 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:50:25.173 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:24 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:50:25.173 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:24 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:50:25.173 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:24 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:50:25.173 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:24 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:50:25.173 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:24 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:50:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:50:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:50:25.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:50:25.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:50:25.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:24 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:50:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:50:25] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-04-30T09:50:26.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:25 smithi026 ceph-mon[27123]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:26.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:25 smithi026 ceph-mon[27123]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:50:26.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:25 smithi026 ceph-mon[27123]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:50:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:25 smithi026 ceph-mon[22362]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:25 smithi026 ceph-mon[22362]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:50:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:25 smithi026 ceph-mon[22362]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:50:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:25 smithi046 ceph-mon[23181]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:25 smithi046 ceph-mon[23181]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:50:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:25 smithi046 ceph-mon[23181]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:50:26.662+0000 7f31c552d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:50:26.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:50:26.986 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return future.result(timeout) 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.__get_result() 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: raise self._exception 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: use_current_daemon_image=reconfig, 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:50:26.987 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:50:26.988 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:26.989 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:26 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:27 smithi026 ceph-mon[22362]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:27 smithi026 ceph-mon[22362]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:50:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:50:28.198 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:50:28.199 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:50:28.200 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:27 smithi026 ceph-mon[27123]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:27 smithi026 ceph-mon[27123]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:50:28.201 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:50:28.202 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-30T09:50:28.203 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:50:28.204 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:27 smithi046 ceph-mon[23181]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:27 smithi046 ceph-mon[23181]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:50:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:50:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:50:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:50:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:50:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return future.result(timeout) 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.__get_result() 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: raise self._exception 2024-04-30T09:50:28.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:50:28.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:50:28.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:50:28.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:50:28.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:50:28.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:50:28.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:50:28.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:50:28.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:50:28.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:50:30.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:29 smithi026 ceph-mon[27123]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:30.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:29 smithi026 ceph-mon[22362]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:29 smithi046 ceph-mon[23181]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:31.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:30 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:30 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:30 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:31 smithi026 ceph-mon[27123]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:31 smithi026 ceph-mon[22362]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:31 smithi046 ceph-mon[23181]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:33 smithi026 ceph-mon[22362]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:33 smithi026 ceph-mon[27123]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:33 smithi046 ceph-mon[23181]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:35.172 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:34 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:34 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:35.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:34 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:35.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:50:35] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-04-30T09:50:36.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:35 smithi026 ceph-mon[27123]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:35 smithi026 ceph-mon[22362]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:35 smithi046 ceph-mon[23181]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:37 smithi026 ceph-mon[27123]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:37 smithi026 ceph-mon[22362]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:37 smithi046 ceph-mon[23181]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:40.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:39 smithi026 ceph-mon[27123]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:39 smithi026 ceph-mon[22362]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:40.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:39 smithi046 ceph-mon[23181]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:41 smithi026 ceph-mon[27123]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:41 smithi026 ceph-mon[22362]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:41 smithi046 ceph-mon[23181]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:44.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:43 smithi026 ceph-mon[22362]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:44.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:43 smithi026 ceph-mon[27123]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:43 smithi046 ceph-mon[23181]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:45.171 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:44 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:45.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:44 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:45.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:44 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:45.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:50:45] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-04-30T09:50:46.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:45 smithi026 ceph-mon[27123]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:45 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:45 smithi026 ceph-mon[22362]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:45 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:45 smithi046 ceph-mon[23181]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:45 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:50:48.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:47 smithi026 ceph-mon[22362]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:48.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:47 smithi026 ceph-mon[27123]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:47 smithi046 ceph-mon[23181]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:50.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:49 smithi026 ceph-mon[22362]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:50.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:49 smithi026 ceph-mon[27123]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:49 smithi046 ceph-mon[23181]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:52.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:51 smithi026 ceph-mon[27123]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:51 smithi026 ceph-mon[22362]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:52.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:51 smithi046 ceph-mon[23181]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:54.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:53 smithi026 ceph-mon[27123]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:54.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:53 smithi026 ceph-mon[22362]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:53 smithi046 ceph-mon[23181]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:54 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:54 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:54 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:50:55.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:50:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:50:55] "GET /metrics HTTP/1.1" 200 37898 "" "Prometheus/2.33.4" 2024-04-30T09:50:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:55 smithi026 ceph-mon[27123]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:55 smithi026 ceph-mon[22362]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:55 smithi046 ceph-mon[23181]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:50:58.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:57 smithi026 ceph-mon[22362]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:58.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:57 smithi026 ceph-mon[27123]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:50:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:57 smithi046 ceph-mon[23181]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:50:59 smithi046 ceph-mon[23181]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:00.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:50:59 smithi026 ceph-mon[27123]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:50:59 smithi026 ceph-mon[22362]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:00 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:01.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:00 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:01.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:00 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:01 smithi046 ceph-mon[23181]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:02.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:01 smithi026 ceph-mon[27123]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:02.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:01 smithi026 ceph-mon[22362]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:03 smithi046 ceph-mon[23181]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:03 smithi026 ceph-mon[27123]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:03 smithi026 ceph-mon[22362]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:04 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:05.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:51:05] "GET /metrics HTTP/1.1" 200 37896 "" "Prometheus/2.33.4" 2024-04-30T09:51:05.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:04 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:04 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:05 smithi046 ceph-mon[23181]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:06.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:05 smithi026 ceph-mon[27123]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:06.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:05 smithi026 ceph-mon[22362]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:07 smithi046 ceph-mon[23181]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:08.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:07 smithi026 ceph-mon[27123]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:08.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:07 smithi026 ceph-mon[22362]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:10.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:09 smithi046 ceph-mon[23181]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:10.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:09 smithi026 ceph-mon[27123]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:10.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:09 smithi026 ceph-mon[22362]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:11 smithi046 ceph-mon[23181]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:12.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:11 smithi026 ceph-mon[27123]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:11 smithi026 ceph-mon[22362]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:14.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:13 smithi046 ceph-mon[23181]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:13 smithi026 ceph-mon[27123]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:13 smithi026 ceph-mon[22362]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:15.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:14 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:15.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:51:15] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-30T09:51:15.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:14 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:15.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:14 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:16.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:15 smithi046 ceph-mon[23181]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:16.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:15 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:16.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:15 smithi026 ceph-mon[27123]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:15 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:15 smithi026 ceph-mon[22362]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:16.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:15 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:17.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:16 smithi026 ceph-mon[27123]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:16 smithi026 ceph-mon[22362]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:16 smithi046 ceph-mon[23181]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:19.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:19 smithi026 ceph-mon[27123]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:19 smithi026 ceph-mon[22362]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:19 smithi046 ceph-mon[23181]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:22.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:21 smithi026 ceph-mon[22362]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:22.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:21 smithi026 ceph-mon[27123]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:21 smithi046 ceph-mon[23181]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:23.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:23 smithi026 ceph-mon[22362]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:23 smithi026 ceph-mon[27123]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:23 smithi046 ceph-mon[23181]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:25.172 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:24 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:24 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:24 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:25.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:25 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:51:25] "GET /metrics HTTP/1.1" 200 37908 "" "Prometheus/2.33.4" 2024-04-30T09:51:26.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:25 smithi026 ceph-mon[27123]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:25 smithi026 ceph-mon[22362]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:25 smithi046 ceph-mon[23181]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:26 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:51:27.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:26 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:51:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:26 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:51:28.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[22362]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:28.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:51:28.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[27123]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:51:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:27 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:51:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:27 smithi046 ceph-mon[23181]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:27 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:51:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:27 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:51:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:27 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:51:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:27 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:51:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:27 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:51:28.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:27 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:51:29.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:28 smithi026 ceph-mon[27123]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:51:29.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:28 smithi026 ceph-mon[27123]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:51:29.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:28 smithi026 ceph-mon[22362]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:51:29.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:28 smithi026 ceph-mon[22362]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:51:29.200 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:28 smithi046 ceph-mon[23181]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:51:29.200 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:28 smithi046 ceph-mon[23181]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:51:29.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:51:29.198+0000 7f31c552d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:51:29.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:51:29.485 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:51:29.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:51:29.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:51:29.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:51:29.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:51:29.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:29.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:29.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return future.result(timeout) 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.__get_result() 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: raise self._exception 2024-04-30T09:51:29.487 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: use_current_daemon_image=reconfig, 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:51:29.488 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:29.489 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:30.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:29 smithi026 ceph-mon[22362]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:30.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:29 smithi026 ceph-mon[27123]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:29 smithi046 ceph-mon[23181]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:30 smithi026 ceph-mon[27123]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:51:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:51:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:51:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:51:31.200 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:30 smithi026 ceph-mon[22362]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:51:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:51:31.202 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:51:31.203 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:51:31.204 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:51:31.205 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:51:31.205 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.205 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:30 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:31.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:51:31.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:51:31.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:51:31.205 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.206 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:30 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:30 smithi046 ceph-mon[23181]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:51:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:51:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:51:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:51:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:51:31.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return future.result(timeout) 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.__get_result() 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: raise self._exception 2024-04-30T09:51:31.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:51:31.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:51:31.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:30 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:31 smithi026 ceph-mon[27123]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:31 smithi026 ceph-mon[22362]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:31 smithi046 ceph-mon[23181]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:34.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:33 smithi026 ceph-mon[27123]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:33 smithi026 ceph-mon[22362]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:33 smithi046 ceph-mon[23181]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:34 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:34 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:34 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:35.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:35 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:51:35] "GET /metrics HTTP/1.1" 200 37904 "" "Prometheus/2.33.4" 2024-04-30T09:51:36.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:35 smithi026 ceph-mon[27123]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:36.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:35 smithi026 ceph-mon[22362]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:35 smithi046 ceph-mon[23181]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:37 smithi026 ceph-mon[27123]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:37 smithi026 ceph-mon[22362]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:37 smithi046 ceph-mon[23181]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:40.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:39 smithi026 ceph-mon[27123]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:39 smithi026 ceph-mon[22362]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:39 smithi046 ceph-mon[23181]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:41 smithi026 ceph-mon[27123]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:41 smithi026 ceph-mon[22362]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:41 smithi046 ceph-mon[23181]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:51:44.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:43 smithi026 ceph-mon[27123]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:44.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:43 smithi026 ceph-mon[22362]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:44.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:43 smithi046 ceph-mon[23181]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:45.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:44 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:45.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:44 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:44 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:45.235 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:45 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:51:45] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-04-30T09:51:46.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:45 smithi026 ceph-mon[27123]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:45 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:45 smithi026 ceph-mon[22362]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:45 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:45 smithi046 ceph-mon[23181]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:45 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:51:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:47 smithi026 ceph-mon[22362]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:48.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:47 smithi026 ceph-mon[27123]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:47 smithi046 ceph-mon[23181]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:49 smithi046 ceph-mon[23181]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:50.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:49 smithi026 ceph-mon[22362]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:50.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:49 smithi026 ceph-mon[27123]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:51 smithi046 ceph-mon[23181]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:52.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:51 smithi026 ceph-mon[22362]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:52.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:51 smithi026 ceph-mon[27123]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:53 smithi046 ceph-mon[23181]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:54.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:53 smithi026 ceph-mon[27123]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:54.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:53 smithi026 ceph-mon[22362]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:54 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:55.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:51:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:51:55] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-04-30T09:51:55.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:54 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:55.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:54 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:51:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:55 smithi046 ceph-mon[23181]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:56.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:55 smithi026 ceph-mon[22362]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:56.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:55 smithi026 ceph-mon[27123]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T09:51:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:57 smithi046 ceph-mon[23181]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:58.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:57 smithi026 ceph-mon[22362]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:51:58.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:57 smithi026 ceph-mon[27123]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:51:59 smithi046 ceph-mon[23181]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:51:59 smithi026 ceph-mon[27123]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:51:59 smithi026 ceph-mon[22362]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:01.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:00 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:01.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:00 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:01.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:00 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:02.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:01 smithi046 ceph-mon[23181]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:02.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:01 smithi026 ceph-mon[22362]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:02.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:01 smithi026 ceph-mon[27123]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:03.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T09:52:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:03 smithi046 ceph-mon[23181]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:04.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:03 smithi026 ceph-mon[27123]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:04.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:03 smithi026 ceph-mon[22362]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (12m) 2m ago 20m 23.1M - 0.23.0 ba2b418f427c 1648d8f22b65 2024-04-30T09:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (12m) 2m ago 20m 52.4M - 8.3.5 dad864ee21e9 9bd49a225988 2024-04-30T09:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 running (6m) 2m ago 19m 53.6M - 3.5 e1d6a67b021e 5bb76058e663 2024-04-30T09:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283 running (14m) 2m ago 25m 495M - 19.0.0-3357-g3a873b0c c50cf5fa633a 53731c91d116 2024-04-30T09:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (9m) 2m ago 26m 72.7M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (27m) 2m ago 27m 71.3M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (25m) 2m ago 25m 52.1M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (25m) 2m ago 25m 53.8M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (12m) 2m ago 21m 17.6M - 1.3.1 1dbe0e931976 b6af5622fe4c 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (12m) 2m ago 21m 17.3M - 1.3.1 1dbe0e931976 23b6576e6bb5 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (24m) 2m ago 24m 52.9M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (24m) 2m ago 24m 52.4M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (24m) 2m ago 24m 52.3M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (23m) 2m ago 23m 50.6M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (23m) 2m ago 23m 55.2M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (22m) 2m ago 22m 51.7M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (22m) 2m ago 22m 51.2M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (22m) 2m ago 22m 52.7M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (2m) 2m ago 21m 51.5M - 2.33.4 514e6a882f6e a48285f81913 2024-04-30T09:52:05.169 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (19m) 2m ago 19m 74.3M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:52:05.170 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (19m) 2m ago 19m 74.5M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:52:05.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:04 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:05.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:04 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:05.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:04 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:05.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:52:05] "GET /metrics HTTP/1.1" 200 37906 "" "Prometheus/2.33.4" 2024-04-30T09:52:05.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ls' 2024-04-30T09:52:06.071 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:05 smithi026 ceph-mon[27123]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:06.072 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:05 smithi026 ceph-mon[22362]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:06.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:05 smithi046 ceph-mon[23181]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:07.285 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:06 smithi026 ceph-mon[22362]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:07.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:06 smithi026 ceph-mon[27123]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:06 smithi046 ceph-mon[23181]: from='client.15426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:07.750 INFO:teuthology.orchestra.run.smithi026.stdout:NAME PORTS RUNNING REFRESHED AGE PLACEMENT 2024-04-30T09:52:07.751 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager ?:9093,9094 1/1 3m ago 21m smithi026=a;count:1 2024-04-30T09:52:07.751 INFO:teuthology.orchestra.run.smithi026.stdout:grafana ?:3000 1/1 2m ago 21m smithi046=a;count:1 2024-04-30T09:52:07.751 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo ?:5000 1/1 3m ago 19m count:1 2024-04-30T09:52:07.751 INFO:teuthology.orchestra.run.smithi026.stdout:mgr 2/2 3m ago 25m smithi026=y;smithi046=x;count:2 2024-04-30T09:52:07.751 INFO:teuthology.orchestra.run.smithi026.stdout:mon 3/3 3m ago 25m smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi046:172.21.15.46=b;count:3 2024-04-30T09:52:07.751 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter ?:9100 2/2 3m ago 21m smithi026=a;smithi046=b;count:2 2024-04-30T09:52:07.751 INFO:teuthology.orchestra.run.smithi026.stdout:osd 8 3m ago - 2024-04-30T09:52:07.751 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus ?:9095 1/1 2m ago 21m smithi046=a;count:1 2024-04-30T09:52:07.751 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo ?:8000 2/2 3m ago 19m count:2 2024-04-30T09:52:08.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:07 smithi026 ceph-mon[27123]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:08.103 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:07 smithi026 ceph-mon[27123]: from='client.15432 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:08.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:07 smithi026 ceph-mon[22362]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:08.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:07 smithi026 ceph-mon[22362]: from='client.15432 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:08.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions' 2024-04-30T09:52:08.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:07 smithi046 ceph-mon[23181]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:08.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:07 smithi046 ceph-mon[23181]: from='client.15432 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:09.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:08 smithi026 ceph-mon[27123]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:09.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:08 smithi026 ceph-mon[22362]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:08 smithi046 ceph-mon[23181]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:52:10.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:52:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:52:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:52:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:52:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-30T09:52:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:52:10.428 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:52:10.428 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:52:11.107 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-04-30T09:52:11.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:11 smithi026 ceph-mon[27123]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:11.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:11 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3883934794' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:11.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:11 smithi026 ceph-mon[22362]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:11.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:11 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3883934794' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:11 smithi046 ceph-mon[23181]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:11 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3883934794' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:13 smithi026 ceph-mon[27123]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:13.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:13 smithi026 ceph-mon[22362]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:13 smithi046 ceph-mon[23181]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:14.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:14 smithi026 ceph-mon[27123]: from='client.15444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:14.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:14 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:14.269 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:14 smithi026 ceph-mon[22362]: from='client.15444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:14.269 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:14 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:14.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:14 smithi046 ceph-mon[23181]: from='client.15444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:14.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:14 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:15.102 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:52:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:15 smithi026 ceph-mon[27123]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:15 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:52:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:15 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:15 smithi026 ceph-mon[22362]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:15 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:52:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:15 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:15 smithi046 ceph-mon[23181]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:15 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:52:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:15 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:15.484 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:52:15] "GET /metrics HTTP/1.1" 200 37903 "" "Prometheus/2.33.4" 2024-04-30T09:52:15.869 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:16.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[27123]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:16.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:16 smithi026 ceph-mon[22362]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:52:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:52:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:52:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:52:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: Reconfiguring iscsi.foo.smithi026.romfrr (dependencies changed)... 2024-04-30T09:52:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:52:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T09:52:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:16.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:16 smithi046 ceph-mon[23181]: Reconfiguring daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:52:17.305+0000 7f31c552d700 -1 log_channel(cephadm) log [ERR] : cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:52:17.735 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:52:17.736 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:52:17.736 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:52:17.736 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:52:17.736 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:52:17.736 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:52:17.736 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:52:17.736 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:17.736 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return future.result(timeout) 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return self.__get_result() 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: raise self._exception 2024-04-30T09:52:17.737 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: use_current_daemon_image=reconfig, 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Creating ceph-iscsi config... 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: Traceback (most recent call last): 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:52:17.738 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: return _run_code(code, main_globals, None, 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: exec(code, run_globals) 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:17.739 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:18.007 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:17 smithi026 ceph-mon[27123]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:18.007 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:17 smithi026 ceph-mon[22362]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:18.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:17 smithi046 ceph-mon[23181]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:18.785 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:52:19.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:18 smithi026 ceph-mon[22362]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:52:19.043 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:52:19.044 INFO:journalctl@ceph.mon.a.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: return future.result(timeout) 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: return self.__get_result() 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: raise self._exception 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:52:19.045 INFO:journalctl@ceph.mon.a.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:52:19.046 INFO:journalctl@ceph.mon.a.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.047 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:18 smithi026 ceph-mon[22362]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:18 smithi026 ceph-mon[27123]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:52:19.048 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:52:19.049 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:52:19.049 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:52:19.049 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:52:19.049 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:52:19.049 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:52:19.049 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:52:19.049 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:52:19.049 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.049 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.050 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:52:19.050 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:52:19.050 INFO:journalctl@ceph.mon.c.smithi026.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:52:19.050 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:52:19.050 INFO:journalctl@ceph.mon.c.smithi026.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:52:19.050 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:52:19.050 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: return future.result(timeout) 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: return self.__get_result() 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: raise self._exception 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: Creating ceph-iscsi config... 2024-04-30T09:52:19.051 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: Traceback (most recent call last): 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: exec(code, run_globals) 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:52:19.052 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:52:19.053 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:52:19.053 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:52:19.053 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:52:19.053 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:52:19.053 INFO:journalctl@ceph.mon.c.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:52:19.053 INFO:journalctl@ceph.mon.c.smithi026.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.053 INFO:journalctl@ceph.mon.c.smithi026.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.053 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:18 smithi026 ceph-mon[27123]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:52:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:18 smithi046 ceph-mon[23181]: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:52:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:52:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:52:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:52:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:52:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:52:19.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:52:19.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:52:19.236 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:52:19.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:52:19.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:52:19.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:52:19.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:52:19.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:52:19.237 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:52:19.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:52:19.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:52:19.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.238 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:52:19.239 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1130, in _check_daemons 2024-04-30T09:52:19.239 INFO:journalctl@ceph.mon.b.smithi046.stdout: self.mgr._daemon_action(daemon_spec, action=action) 2024-04-30T09:52:19.239 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 2463, in _daemon_action 2024-04-30T09:52:19.239 INFO:journalctl@ceph.mon.b.smithi046.stdout: CephadmServe(self)._create_daemon(daemon_spec, reconfig=(action == 'reconfig'))) 2024-04-30T09:52:19.239 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/module.py", line 784, in wait_async 2024-04-30T09:52:19.239 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.event_loop.get_result(coro, timeout) 2024-04-30T09:52:19.239 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/ssh.py", line 136, in get_result 2024-04-30T09:52:19.240 INFO:journalctl@ceph.mon.b.smithi046.stdout: return future.result(timeout) 2024-04-30T09:52:19.240 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 432, in result 2024-04-30T09:52:19.240 INFO:journalctl@ceph.mon.b.smithi046.stdout: return self.__get_result() 2024-04-30T09:52:19.240 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/lib64/python3.6/concurrent/futures/_base.py", line 384, in __get_result 2024-04-30T09:52:19.240 INFO:journalctl@ceph.mon.b.smithi046.stdout: raise self._exception 2024-04-30T09:52:19.240 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1393, in _create_daemon 2024-04-30T09:52:19.240 INFO:journalctl@ceph.mon.b.smithi046.stdout: use_current_daemon_image=reconfig, 2024-04-30T09:52:19.240 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1710, in _run_cephadm 2024-04-30T09:52:19.241 INFO:journalctl@ceph.mon.b.smithi046.stdout: f'cephadm exited with an error code: {code}, stderr: {err}') 2024-04-30T09:52:19.241 INFO:journalctl@ceph.mon.b.smithi046.stdout: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr: Reconfig daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T09:52:19.241 INFO:journalctl@ceph.mon.b.smithi046.stdout: Creating ceph-iscsi config... 2024-04-30T09:52:19.241 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T09:52:19.241 INFO:journalctl@ceph.mon.b.smithi046.stdout: Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T09:52:19.241 INFO:journalctl@ceph.mon.b.smithi046.stdout: Non-zero exit code 1 from systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T09:52:19.241 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.241 INFO:journalctl@ceph.mon.b.smithi046.stdout: systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.241 INFO:journalctl@ceph.mon.b.smithi046.stdout: Traceback (most recent call last): 2024-04-30T09:52:19.242 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T09:52:19.242 INFO:journalctl@ceph.mon.b.smithi046.stdout: return _run_code(code, main_globals, None, 2024-04-30T09:52:19.242 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T09:52:19.242 INFO:journalctl@ceph.mon.b.smithi046.stdout: exec(code, run_globals) 2024-04-30T09:52:19.242 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T09:52:19.242 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T09:52:19.242 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T09:52:19.242 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T09:52:19.243 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T09:52:19.243 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1123, in deploy_daemon 2024-04-30T09:52:19.243 INFO:journalctl@ceph.mon.b.smithi046.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T09:52:19.243 INFO:journalctl@ceph.mon.b.smithi046.stdout: RuntimeError: Failed command: systemctl restart ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T09:52:19.243 INFO:journalctl@ceph.mon.b.smithi046.stdout: See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T09:52:19.243 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:18 smithi046 ceph-mon[23181]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:52:19.506 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (12m) 3m ago 20m 23.1M - 0.23.0 ba2b418f427c 1648d8f22b65 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (12m) 2m ago 20m 52.4M - 8.3.5 dad864ee21e9 9bd49a225988 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 running (6m) 3m ago 19m 53.6M - 3.5 e1d6a67b021e 5bb76058e663 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283 running (15m) 2m ago 25m 495M - 19.0.0-3357-g3a873b0c c50cf5fa633a 53731c91d116 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (9m) 3m ago 27m 72.7M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (27m) 3m ago 27m 71.3M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (25m) 2m ago 25m 52.1M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (25m) 3m ago 25m 53.8M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (12m) 3m ago 21m 17.6M - 1.3.1 1dbe0e931976 b6af5622fe4c 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (12m) 2m ago 21m 17.3M - 1.3.1 1dbe0e931976 23b6576e6bb5 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (25m) 3m ago 25m 52.9M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (24m) 3m ago 24m 52.4M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (24m) 3m ago 24m 52.3M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:52:19.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (23m) 3m ago 23m 50.6M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:52:19.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (23m) 2m ago 23m 55.2M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:52:19.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (23m) 2m ago 23m 51.7M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:52:19.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (22m) 2m ago 22m 51.2M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:52:19.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (22m) 2m ago 22m 52.7M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:52:19.508 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (3m) 2m ago 21m 51.5M - 2.33.4 514e6a882f6e a48285f81913 2024-04-30T09:52:19.508 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (20m) 3m ago 20m 74.3M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:52:19.508 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (19m) 2m ago 19m 74.5M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:52:19.960 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:52:19.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:19.961 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:52:19.961 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-30T09:52:19.961 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:52:19.961 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:52:19.961 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:52:20.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[27123]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:20.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[27123]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:20.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[27123]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[27123]: osdmap e112: 8 total, 8 up, 8 in 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[22362]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[22362]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[22362]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-30T09:52:20.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:19 smithi026 ceph-mon[22362]: osdmap e112: 8 total, 8 up, 8 in 2024-04-30T09:52:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:19 smithi046 ceph-mon[23181]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 96 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:19 smithi046 ceph-mon[23181]: from='client.15450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:19 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' 2024-04-30T09:52:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:19 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:19 smithi046 ceph-mon[23181]: from='mgr.24986 172.21.15.46:0/1339195480' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-30T09:52:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:19 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-30T09:52:20.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:19 smithi046 ceph-mon[23181]: osdmap e112: 8 total, 8 up, 8 in 2024-04-30T09:52:20.309 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:52:20.309 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:52:20.309 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:52:20.309 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-30T09:52:20.309 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-30T09:52:20.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-04-30T09:52:20.309 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:52:20.309 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-30T09:52:20.309 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 image", 2024-04-30T09:52:20.310 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:52:20.310 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:52:21.196 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:20] ENGINE Bus STOPPING 2024-04-30T09:52:21.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:52:21.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:20] ENGINE Bus STOPPED 2024-04-30T09:52:21.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:20] ENGINE Bus STARTING 2024-04-30T09:52:21.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:21] ENGINE Serving on http://:::9283 2024-04-30T09:52:21.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:21] ENGINE Bus STARTED 2024-04-30T09:52:21.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='client.25354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:21.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='client.15462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:21.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:52:21.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: Failing over to other MGR 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1717568428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: Standby manager daemon x started 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.24986 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: mgrmap e37: y(active, starting, since 1.00018s), standbys: x 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:52:21.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='client.25354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='client.15462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:52:21.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: Failing over to other MGR 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1717568428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: Standby manager daemon x started 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.24986 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: mgrmap e37: y(active, starting, since 1.00018s), standbys: x 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:52:21.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:52:21.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:52:21.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:52:21.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:52:21.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:52:21.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:52:21.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:52:21.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:52:21.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:20 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:52:21.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:52:20.893+0000 7f323372f700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-30T09:52:21.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ignoring --setuser ceph since I am not root 2024-04-30T09:52:21.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: ignoring --setgroup ceph since I am not root 2024-04-30T09:52:21.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mgr[43498]: -- 172.21.15.46:0/4291144334 <== mon.2 v2:172.21.15.46:3300/0 3 ==== mon_map magic: 0 ==== 413+0+0 (secure 0 0 0) 0x562a216044e0 con 0x562a215dac00 2024-04-30T09:52:21.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mgr[43498]: -- 172.21.15.46:0/4291144334 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562a216065a0 con 0x562a215dac00 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='client.25354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='client.15462 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: Upgrade: Need to upgrade myself (mgr.x) 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: Failing over to other MGR 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1717568428' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: Standby manager daemon x started 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.24986 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-30T09:52:21.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: mgrmap e37: y(active, starting, since 1.00018s), standbys: x 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:52:21.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:52:21.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:20 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:52:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[22362]: Manager daemon y is now available 2024-04-30T09:52:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:52:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:52:22.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[27123]: Manager daemon y is now available 2024-04-30T09:52:22.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:22.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:22.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:52:22.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:21 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:52:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:21 smithi046 ceph-mon[23181]: Manager daemon y is now available 2024-04-30T09:52:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:21 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:21 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:21 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:52:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:21 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:52:23.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:22 smithi026 ceph-mon[27123]: mgrmap e38: y(active, since 2s), standbys: x 2024-04-30T09:52:23.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:22 smithi026 ceph-mon[27123]: [30/Apr/2024:09:52:21] ENGINE Bus STARTING 2024-04-30T09:52:23.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:22 smithi026 ceph-mon[27123]: [30/Apr/2024:09:52:22] ENGINE Serving on http://172.21.15.26:8765 2024-04-30T09:52:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:22 smithi026 ceph-mon[22362]: mgrmap e38: y(active, since 2s), standbys: x 2024-04-30T09:52:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:22 smithi026 ceph-mon[22362]: [30/Apr/2024:09:52:21] ENGINE Bus STARTING 2024-04-30T09:52:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:22 smithi026 ceph-mon[22362]: [30/Apr/2024:09:52:22] ENGINE Serving on http://172.21.15.26:8765 2024-04-30T09:52:23.209 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:22 smithi046 ceph-mon[23181]: mgrmap e38: y(active, since 2s), standbys: x 2024-04-30T09:52:23.209 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:22 smithi046 ceph-mon[23181]: [30/Apr/2024:09:52:21] ENGINE Bus STARTING 2024-04-30T09:52:23.209 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:22 smithi046 ceph-mon[23181]: [30/Apr/2024:09:52:22] ENGINE Serving on http://172.21.15.26:8765 2024-04-30T09:52:23.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:23 smithi026 ceph-mon[27123]: [30/Apr/2024:09:52:22] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:52:23.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:23 smithi026 ceph-mon[27123]: [30/Apr/2024:09:52:22] ENGINE Bus STARTED 2024-04-30T09:52:23.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:23 smithi026 ceph-mon[27123]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:52:23.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:23 smithi026 ceph-mon[22362]: [30/Apr/2024:09:52:22] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:52:23.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:23 smithi026 ceph-mon[22362]: [30/Apr/2024:09:52:22] ENGINE Bus STARTED 2024-04-30T09:52:23.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:23 smithi026 ceph-mon[22362]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:52:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:23 smithi046 ceph-mon[23181]: [30/Apr/2024:09:52:22] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:52:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:23 smithi046 ceph-mon[23181]: [30/Apr/2024:09:52:22] ENGINE Bus STARTED 2024-04-30T09:52:24.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:23 smithi046 ceph-mon[23181]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:52:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:24 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:24 smithi026 ceph-mon[27123]: mgrmap e39: y(active, since 4s), standbys: x 2024-04-30T09:52:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:24 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:24 smithi026 ceph-mon[22362]: mgrmap e39: y(active, since 4s), standbys: x 2024-04-30T09:52:25.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:24 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:25.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:24 smithi046 ceph-mon[23181]: mgrmap e39: y(active, since 4s), standbys: x 2024-04-30T09:52:26.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:25 smithi026 ceph-mon[27123]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:52:26.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:25 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:26.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:25 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:25 smithi026 ceph-mon[22362]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:52:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:25 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:26.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:25 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:25 smithi046 ceph-mon[23181]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:25 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:26.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:25 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:27 smithi046 ceph-mon[23181]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:52:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[27123]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:52:27.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[22362]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:52:27.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:27.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:28.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:52:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:52:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:52:28.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:52:28.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:52:28.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:52:28.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:52:28.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:28.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:28.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:28.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:52:28.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:52:28.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:52:28.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:52:28.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:52:28.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:52:28.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:28 smithi026 ceph-mon[22362]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:52:28.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:52:28] ENGINE Bus STARTING 2024-04-30T09:52:28.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: CherryPy Checker: 2024-04-30T09:52:28.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: The Application mounted at '' has an empty config. 2024-04-30T09:52:28.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: 2024-04-30T09:52:28.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:52:28] ENGINE Serving on http://:::9283 2024-04-30T09:52:28.734 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[43494]: [30/Apr/2024:09:52:28] ENGINE Bus STARTED 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:52:28.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:28 smithi046 ceph-mon[23181]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:52:29.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: Standby manager daemon x restarted 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: Standby manager daemon x started 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 0 B/s wr, 111 op/s 2024-04-30T09:52:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[27123]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: Standby manager daemon x restarted 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: Standby manager daemon x started 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 0 B/s wr, 111 op/s 2024-04-30T09:52:29.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:52:29.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:29 smithi026 ceph-mon[22362]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:52:29.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:52:29.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:52:29.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: Standby manager daemon x restarted 2024-04-30T09:52:29.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: Standby manager daemon x started 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1740847559' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 0 B/s wr, 111 op/s 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: Reconfiguring prometheus.a (dependencies changed)... 2024-04-30T09:52:29.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:29 smithi046 ceph-mon[23181]: Reconfiguring daemon prometheus.a on smithi046 2024-04-30T09:52:30.593 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:30 smithi046 ceph-mon[23181]: mgrmap e40: y(active, since 9s), standbys: x 2024-04-30T09:52:30.593 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:30 smithi046 ceph-mon[23181]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T09:52:30.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:30 smithi026 ceph-mon[27123]: mgrmap e40: y(active, since 9s), standbys: x 2024-04-30T09:52:30.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:30 smithi026 ceph-mon[27123]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T09:52:30.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:30 smithi026 ceph-mon[22362]: mgrmap e40: y(active, since 9s), standbys: x 2024-04-30T09:52:30.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:30 smithi026 ceph-mon[22362]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T09:52:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:31 smithi026 ceph-mon[27123]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 0 B/s wr, 85 op/s 2024-04-30T09:52:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:31 smithi026 ceph-mon[22362]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 0 B/s wr, 85 op/s 2024-04-30T09:52:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:31 smithi046 ceph-mon[23181]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 0 B/s wr, 85 op/s 2024-04-30T09:52:32.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 systemd[1]: Stopping Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:52:32.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 bash[58002]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:52:32.616 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-30T09:52:32.616 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-30T09:52:32.616 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-30T09:52:32.616 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-30T09:52:32.616 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-30T09:52:32.616 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-30T09:52:32.616 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-30T09:52:32.617 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-30T09:52:32.617 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-30T09:52:32.617 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-30T09:52:32.617 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-30T09:52:32.617 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[55217]: ts=2024-04-30T09:52:32.330Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-30T09:52:32.617 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 podman[58015]: 2024-04-30 09:52:32.340517761 +0000 UTC m=+0.225586434 container died a48285f81913d73589aa09804e26d7ca63e646c97b554b9a7fb06cb10b1c13b0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:32.981 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 podman[58015]: 2024-04-30 09:52:32.76115415 +0000 UTC m=+0.646222816 container cleanup a48285f81913d73589aa09804e26d7ca63e646c97b554b9a7fb06cb10b1c13b0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:32.982 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 bash[58015]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a 2024-04-30T09:52:32.982 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 bash[58044]: Error: no container with name or ID "ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus.a" found: no such container 2024-04-30T09:52:33.234 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 podman[58029]: 2024-04-30 09:52:32.981210982 +0000 UTC m=+0.637545187 container remove a48285f81913d73589aa09804e26d7ca63e646c97b554b9a7fb06cb10b1c13b0 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:33.237 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-30T09:52:33.237 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 systemd[1]: Stopped Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:52:33.237 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:32 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.425s CPU time. 2024-04-30T09:52:33.237 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 systemd[1]: Starting Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:52:33.237 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 podman[58110]: 2024-04-30 09:52:33.126772503 +0000 UTC m=+0.019286868 image pull 514e6a882f6e74806a5856468489eeff8d7106095557578da96935e4d0ba4d9d quay.io/prometheus/prometheus:v2.33.4 2024-04-30T09:52:33.543 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 podman[58110]: 2024-04-30 09:52:33.248940006 +0000 UTC m=+0.141454393 container create 571af4167709876b57e7a4ad795d499f0edad561d3925d572493cb07fb0a20a7 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:33.543 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 podman[58110]: 2024-04-30 09:52:33.451178871 +0000 UTC m=+0.343693216 container init 571af4167709876b57e7a4ad795d499f0edad561d3925d572493cb07fb0a20a7 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 podman[58110]: 2024-04-30 09:52:33.45383582 +0000 UTC m=+0.346350178 container start 571af4167709876b57e7a4ad795d499f0edad561d3925d572493cb07fb0a20a7 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.477Z caller=main.go:475 level=info msg="No time or size retention was set so using the default time retention" duration=15d 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.477Z caller=main.go:512 level=info msg="Starting Prometheus" version="(version=2.33.4, branch=HEAD, revision=83032011a5d3e6102624fe58241a374a7201fee8)" 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.477Z caller=main.go:517 level=info build_context="(go=go1.17.7, user=root@d13bf69e7be8, date=20220222-16:51:28)" 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.477Z caller=main.go:518 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi046 (none))" 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.477Z caller=main.go:519 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.477Z caller=main.go:520 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.479Z caller=web.go:570 level=info component=web msg="Start listening for connections" address=:9095 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.480Z caller=main.go:923 level=info msg="Starting TSDB ..." 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.481Z caller=tls_config.go:195 level=info component=web msg="TLS is disabled." http2=false 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.482Z caller=head.go:493 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.482Z caller=head.go:527 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.647µs 2024-04-30T09:52:33.544 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.482Z caller=head.go:533 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-30T09:52:33.545 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 bash[58110]: 571af4167709876b57e7a4ad795d499f0edad561d3925d572493cb07fb0a20a7 2024-04-30T09:52:33.910 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:33] ENGINE Bus STOPPING 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:33] ENGINE Bus STOPPED 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:33] ENGINE Bus STARTING 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:33] ENGINE Serving on http://:::9283 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:52:33] ENGINE Bus STARTED 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[22362]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 0 B/s wr, 69 op/s 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:52:33.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:52:33.912 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 systemd[1]: Started Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:52:33.912 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.681Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=5 2024-04-30T09:52:34.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-mon[23181]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 0 B/s wr, 69 op/s 2024-04-30T09:52:34.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:34.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:34.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:52:34.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:52:34.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:34.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:34.168 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:33.914Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=5 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.132Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=5 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.136Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=5 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.138Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=5 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.138Z caller=head.go:604 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=5 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.139Z caller=head.go:610 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=80.584µs wal_replay_duration=656.110536ms total_replay_duration=656.203112ms 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.143Z caller=main.go:944 level=info fs_type=XFS_SUPER_MAGIC 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.143Z caller=main.go:947 level=info msg="TSDB started" 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.143Z caller=main.go:1128 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.167Z caller=main.go:1165 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.742402ms db_storage=863ns remote_storage=1.16µs web_handler=388ns query_engine=714ns scrape=351.886µs scrape_sd=91.286µs notify=10.511µs notify_sd=9.226µs rules=22.954415ms 2024-04-30T09:52:34.169 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:52:34.167Z caller=main.go:896 level=info msg="Server is ready to receive web requests." 2024-04-30T09:52:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[27123]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 0 B/s wr, 69 op/s 2024-04-30T09:52:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:52:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:52:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:33 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:34 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:52:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:34 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:52:35.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:34 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:34 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:52:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:34 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:52:35.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:34 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:52:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi046.front.sepia.ceph.com:9095"}]: dispatch 2024-04-30T09:52:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:34 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:36.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:35 smithi046 ceph-mon[23181]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-30T09:52:36.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:35 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:52:36.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:35 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:52:36.168 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:35 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:36.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:35 smithi026 ceph-mon[27123]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-30T09:52:36.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:35 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:52:36.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:35 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:52:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:35 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:35 smithi026 ceph-mon[22362]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-30T09:52:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:35 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:52:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:35 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:52:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:35 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:37.146 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:36 smithi046 ceph-mon[23181]: Upgrade: Updating mgr.x 2024-04-30T09:52:37.146 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:36 smithi046 ceph-mon[23181]: Deploying daemon mgr.x on smithi046 2024-04-30T09:52:37.147 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:36 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:37.147 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:36 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:37.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:36 smithi026 ceph-mon[27123]: Upgrade: Updating mgr.x 2024-04-30T09:52:37.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:36 smithi026 ceph-mon[27123]: Deploying daemon mgr.x on smithi046 2024-04-30T09:52:37.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:36 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:37.300 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:36 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:37.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:36 smithi026 ceph-mon[22362]: Upgrade: Updating mgr.x 2024-04-30T09:52:37.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:36 smithi026 ceph-mon[22362]: Deploying daemon mgr.x on smithi046 2024-04-30T09:52:37.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:36 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:37.300 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:36 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:37.427 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:37 smithi046 systemd[1]: Stopping Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:52:37.713 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:37 smithi046 podman[58458]: 2024-04-30 09:52:37.426558133 +0000 UTC m=+0.136012201 container died 53731c91d116a3f117c52a188ae444f1a71305b4efbc10948221f29ceab62a7d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux ) 2024-04-30T09:52:37.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:37 smithi046 podman[58458]: 2024-04-30 09:52:37.820209849 +0000 UTC m=+0.529663901 container cleanup 53731c91d116a3f117c52a188ae444f1a71305b4efbc10948221f29ceab62a7d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T09:52:37.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:37 smithi046 bash[58458]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x 2024-04-30T09:52:38.369 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:38 smithi046 ceph-mon[23181]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-30T09:52:38.369 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 podman[58473]: 2024-04-30 09:52:38.055290143 +0000 UTC m=+0.626628575 container remove 53731c91d116a3f117c52a188ae444f1a71305b4efbc10948221f29ceab62a7d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-30T09:52:38.370 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-30T09:52:38.370 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-30T09:52:38.370 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 systemd[1]: Stopped Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:52:38.370 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.x.service: Consumed 1min 2.409s CPU time. 2024-04-30T09:52:38.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:38 smithi026 ceph-mon[22362]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-30T09:52:38.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:38 smithi026 ceph-mon[27123]: pgmap v11: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-30T09:52:38.696 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 systemd[1]: Starting Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:52:38.696 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 podman[58602]: 2024-04-30 09:52:38.488556548 +0000 UTC m=+0.017965217 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:52:38.696 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 podman[58602]: 2024-04-30 09:52:38.589820676 +0000 UTC m=+0.119229374 container create 40603027226c3be2bca8436165050e9d298b31430713b09ce0e6f4b26eb229fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-30T09:52:39.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 podman[58602]: 2024-04-30 09:52:38.817378476 +0000 UTC m=+0.346787138 container init 40603027226c3be2bca8436165050e9d298b31430713b09ce0e6f4b26eb229fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-04-30T09:52:39.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 podman[58602]: 2024-04-30 09:52:38.820047926 +0000 UTC m=+0.349456595 container start 40603027226c3be2bca8436165050e9d298b31430713b09ce0e6f4b26eb229fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-30T09:52:39.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 ceph-mgr[58621]: -- 172.21.15.46:0/3858901212 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564ff059a5a0 con 0x564ff056ec00 2024-04-30T09:52:39.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 bash[58602]: 40603027226c3be2bca8436165050e9d298b31430713b09ce0e6f4b26eb229fa 2024-04-30T09:52:39.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:38 smithi046 systemd[1]: Started Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:52:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:39 smithi046 ceph-mon[23181]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-30T09:52:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:39 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:39 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:39 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:40.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:39 smithi026 ceph-mon[27123]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-30T09:52:40.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:39 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:40.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:39 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:40.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:39 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:40.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:39 smithi026 ceph-mon[22362]: pgmap v12: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 0 B/s wr, 63 op/s 2024-04-30T09:52:40.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:39 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:40.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:39 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:40.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:39 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:42.294 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:41 smithi046 ceph-mon[23181]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:42.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:41 smithi026 ceph-mon[27123]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:42.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:41 smithi026 ceph-mon[22362]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:44.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:43 smithi046 ceph-mon[23181]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:44.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:44.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:44.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:43 smithi026 ceph-mon[22362]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:43 smithi026 ceph-mon[27123]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:52:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:45.279 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:45 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:45.279 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:45 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:45.280 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:45 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:45.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:45 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:45.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:45 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:45.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:45 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:45.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:45 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:45.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:45 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:45.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:45 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:46.278 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:46 smithi046 ceph-mon[23181]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:46.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:46 smithi026 ceph-mon[27123]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:46.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:46 smithi026 ceph-mon[22362]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:46.595 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: [30/Apr/2024:09:52:46] ENGINE Bus STARTING 2024-04-30T09:52:46.595 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: CherryPy Checker: 2024-04-30T09:52:46.595 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: The Application mounted at '' has an empty config. 2024-04-30T09:52:46.595 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: 2024-04-30T09:52:46.595 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: [30/Apr/2024:09:52:46] ENGINE Serving on http://:::9283 2024-04-30T09:52:46.595 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:52:46 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: [30/Apr/2024:09:52:46] ENGINE Bus STARTED 2024-04-30T09:52:46.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:52:46] "GET /metrics HTTP/1.1" 200 37980 "" "Prometheus/2.33.4" 2024-04-30T09:52:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[27123]: Standby manager daemon x restarted 2024-04-30T09:52:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[27123]: Standby manager daemon x started 2024-04-30T09:52:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:52:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:52:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:52:47.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:52:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[22362]: Standby manager daemon x restarted 2024-04-30T09:52:47.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[22362]: Standby manager daemon x started 2024-04-30T09:52:47.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:52:47.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:52:47.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:52:47.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:47 smithi026 ceph-mon[22362]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:52:47.392 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:47 smithi046 ceph-mon[23181]: Standby manager daemon x restarted 2024-04-30T09:52:47.392 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:47 smithi046 ceph-mon[23181]: Standby manager daemon x started 2024-04-30T09:52:47.392 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:47 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:52:47.392 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:47 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:52:47.392 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:47 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:52:47.393 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:47 smithi046 ceph-mon[23181]: from='mgr.? 172.21.15.46:0/1266248560' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:52:48.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:48 smithi026 ceph-mon[27123]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:48.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:48 smithi026 ceph-mon[27123]: mgrmap e41: y(active, since 27s), standbys: x 2024-04-30T09:52:48.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:48 smithi026 ceph-mon[22362]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:48.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:48 smithi026 ceph-mon[22362]: mgrmap e41: y(active, since 27s), standbys: x 2024-04-30T09:52:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:48 smithi046 ceph-mon[23181]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:52:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:48 smithi046 ceph-mon[23181]: mgrmap e41: y(active, since 27s), standbys: x 2024-04-30T09:52:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:49.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:49 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:52:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:49.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:49 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:50.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all mgr 2024-04-30T09:52:50.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all crash 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all mds 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all nfs 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all mgr 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all crash 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all mds 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T09:52:50.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T09:52:50.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all nfs 2024-04-30T09:52:50.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:50 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:52:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:50 smithi046 ceph-mon[23181]: Upgrade: Setting container_image for all mgr 2024-04-30T09:52:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:50 smithi046 ceph-mon[23181]: Upgrade: Setting container_image for all crash 2024-04-30T09:52:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:50 smithi046 ceph-mon[23181]: Upgrade: Setting container_image for all mds 2024-04-30T09:52:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:50 smithi046 ceph-mon[23181]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T09:52:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:50 smithi046 ceph-mon[23181]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T09:52:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:50 smithi046 ceph-mon[23181]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T09:52:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:50 smithi046 ceph-mon[23181]: Upgrade: Setting container_image for all nfs 2024-04-30T09:52:50.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:50 smithi046 ceph-mon[23181]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:52:50.690 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:52:50.947 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:50 smithi026 systemd[1]: Stopping Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:52:51.259 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:51 smithi026 ceph-mon[27123]: Upgrade: Updating node-exporter.a (1/2) 2024-04-30T09:52:51.259 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:51 smithi026 ceph-mon[27123]: Deploying daemon node-exporter.a on smithi026 2024-04-30T09:52:51.259 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:51 smithi026 ceph-mon[27123]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:51.259 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:51 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:51.260 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:50 smithi026 podman[73859]: 2024-04-30 09:52:50.960108157 +0000 UTC m=+0.135728132 container died b6af5622fe4c99e5282e593ca75798b6601decd7a73deca8c29d8b11d9c82066 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:51.260 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:51 smithi026 ceph-mon[22362]: Upgrade: Updating node-exporter.a (1/2) 2024-04-30T09:52:51.260 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:51 smithi026 ceph-mon[22362]: Deploying daemon node-exporter.a on smithi026 2024-04-30T09:52:51.260 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:51 smithi026 ceph-mon[22362]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:51.260 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:51 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (13m) 25s ago 21m 22.8M - 0.23.0 ba2b418f427c 1648d8f22b65 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (13m) 7s ago 21m 52.2M - 8.3.5 dad864ee21e9 9bd49a225988 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 unknown 25s ago 20m 58.2M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (12s) 7s ago 25m 66.2M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (10m) 25s ago 27m 476M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (27m) 25s ago 27m 76.0M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (26m) 7s ago 26m 55.4M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (26m) 25s ago 26m 57.9M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (13m) 25s ago 22m 17.5M - 1.3.1 1dbe0e931976 b6af5622fe4c 2024-04-30T09:52:51.406 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (13m) 7s ago 21m 18.6M - 1.3.1 1dbe0e931976 23b6576e6bb5 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (25m) 25s ago 25m 53.6M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (25m) 25s ago 25m 53.0M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (24m) 25s ago 24m 52.6M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (24m) 25s ago 24m 52.3M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (24m) 7s ago 24m 58.0M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (23m) 7s ago 23m 52.9M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (23m) 7s ago 23m 51.8M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (22m) 7s ago 22m 54.1M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (18s) 7s ago 22m 53.7M - 2.33.4 514e6a882f6e 571af4167709 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (20m) 25s ago 20m 75.2M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:52:51.407 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (20m) 7s ago 20m 75.4M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:52:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:51 smithi046 ceph-mon[23181]: Upgrade: Updating node-exporter.a (1/2) 2024-04-30T09:52:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:51 smithi046 ceph-mon[23181]: Deploying daemon node-exporter.a on smithi026 2024-04-30T09:52:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:51 smithi046 ceph-mon[23181]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:51 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:52:51.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:51 smithi026 podman[73859]: 2024-04-30 09:52:51.366155519 +0000 UTC m=+0.541775494 container cleanup b6af5622fe4c99e5282e593ca75798b6601decd7a73deca8c29d8b11d9c82066 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:51.609 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:51 smithi026 bash[73859]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a 2024-04-30T09:52:51.851 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:52:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:52:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:52:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:52:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:52:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:51.852 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:52:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:52:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:52:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:52:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:52:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:52:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:52:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-30T09:52:51.853 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:52:51.854 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:52:51.854 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:52:51.936 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:51 smithi026 podman[73873]: 2024-04-30 09:52:51.609144271 +0000 UTC m=+0.649158686 container remove b6af5622fe4c99e5282e593ca75798b6601decd7a73deca8c29d8b11d9c82066 (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:51.936 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:51 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-30T09:52:51.936 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:51 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-30T09:52:51.936 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:51 smithi026 systemd[1]: Stopped Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:52:51.936 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:51 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 5.499s CPU time. 2024-04-30T09:52:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:52 smithi026 ceph-mon[22362]: from='client.25414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:52 smithi026 ceph-mon[22362]: from='client.15522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:52 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1657185178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:52.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:52 smithi026 ceph-mon[27123]: from='client.25414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:52.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:52 smithi026 ceph-mon[27123]: from='client.15522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:52.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:52 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1657185178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:52.198 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:51 smithi026 systemd[1]: Starting Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:52:52.198 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:52 smithi026 bash[74075]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-30T09:52:52.219 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:52:52.219 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:52:52.219 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:52:52.219 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-30T09:52:52.219 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-30T09:52:52.219 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-04-30T09:52:52.219 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:52:52.220 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-30T09:52:52.220 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading node-exporter daemons", 2024-04-30T09:52:52.220 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:52:52.220 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:52:52.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:52 smithi046 ceph-mon[23181]: from='client.25414 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:52.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:52 smithi046 ceph-mon[23181]: from='client.15522 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:52.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:52 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1657185178' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:52:53.103 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:52 smithi026 bash[74075]: Getting image source signatures 2024-04-30T09:52:53.103 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:52 smithi026 bash[74075]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-30T09:52:53.103 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:52 smithi026 bash[74075]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-30T09:52:53.103 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:52 smithi026 bash[74075]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-30T09:52:53.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:53 smithi026 ceph-mon[27123]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:53.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:53 smithi026 ceph-mon[27123]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:53 smithi026 ceph-mon[22362]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:53 smithi026 ceph-mon[22362]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:53 smithi046 ceph-mon[23181]: from='client.25426 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:53 smithi046 ceph-mon[23181]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:54.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:54 smithi026 ceph-mon[22362]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:54 smithi026 ceph-mon[27123]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:54 smithi046 ceph-mon[23181]: from='client.15540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:52:55.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:55 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:55.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:55 smithi026 ceph-mon[27123]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:55.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:55 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:55.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:55 smithi026 ceph-mon[22362]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:55 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:52:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:55 smithi046 ceph-mon[23181]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:55.946 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:55 smithi026 bash[74075]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-30T09:52:55.946 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:55 smithi026 bash[74075]: Writing manifest to image destination 2024-04-30T09:52:56.358 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 podman[74075]: 2024-04-30 09:52:55.999062392 +0000 UTC m=+3.957816414 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-30T09:52:56.358 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 podman[74075]: 2024-04-30 09:52:56.147262228 +0000 UTC m=+4.106016247 container create 449290d188b2310e5321ad27e3df78a06cba2ba6184d15fab90666e113e335e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:56.358 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 podman[74075]: 2024-04-30 09:52:56.357601831 +0000 UTC m=+4.316355850 container init 449290d188b2310e5321ad27e3df78a06cba2ba6184d15fab90666e113e335e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:56.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:52:56] "GET /metrics HTTP/1.1" 200 37978 "" "Prometheus/2.33.4" 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 podman[74075]: 2024-04-30 09:52:56.360869058 +0000 UTC m=+4.319623084 container start 449290d188b2310e5321ad27e3df78a06cba2ba6184d15fab90666e113e335e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.365Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.365Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.365Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=node_exporter.go:117 level=info collector=arp 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-30T09:52:56.697 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-30T09:52:56.698 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-30T09:52:56.698 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-30T09:52:56.698 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-30T09:52:56.698 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.366Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-30T09:52:56.698 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-30T09:52:56.698 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=edac 2024-04-30T09:52:56.698 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-30T09:52:56.699 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=os 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=stat 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-30T09:52:56.700 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=time 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=uname 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a[74126]: ts=2024-04-30T09:52:56.367Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 bash[74075]: 449290d188b2310e5321ad27e3df78a06cba2ba6184d15fab90666e113e335e7 2024-04-30T09:52:56.701 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 09:52:56 smithi026 systemd[1]: Started Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:52:57.170 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:56 smithi046 ceph-mon[23181]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:57.170 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:56 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:57.170 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:56 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-mon[27123]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-mon[22362]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:52:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:56 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:52:58.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:58 smithi026 ceph-mon[27123]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:52:58.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:58 smithi026 ceph-mon[22362]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:52:58.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:58 smithi046 ceph-mon[23181]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:52:58.984 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:52:58 smithi046 systemd[1]: Stopping Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:52:59.423 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:59 smithi046 ceph-mon[23181]: Upgrade: Updating node-exporter.b (2/2) 2024-04-30T09:52:59.423 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:52:59 smithi046 ceph-mon[23181]: Deploying daemon node-exporter.b on smithi046 2024-04-30T09:52:59.423 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:52:59 smithi046 podman[59995]: 2024-04-30 09:52:59.165276994 +0000 UTC m=+0.111782751 container died 23b6576e6bb541b3f77460e4ef986aa6c2771b3d737e75768dd95a18f2b9197c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:52:59.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:59 smithi026 ceph-mon[22362]: Upgrade: Updating node-exporter.b (2/2) 2024-04-30T09:52:59.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:52:59 smithi026 ceph-mon[22362]: Deploying daemon node-exporter.b on smithi046 2024-04-30T09:52:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:59 smithi026 ceph-mon[27123]: Upgrade: Updating node-exporter.b (2/2) 2024-04-30T09:52:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:52:59 smithi026 ceph-mon[27123]: Deploying daemon node-exporter.b on smithi046 2024-04-30T09:52:59.734 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:52:59 smithi046 podman[59995]: 2024-04-30 09:52:59.512866902 +0000 UTC m=+0.459372647 container cleanup 23b6576e6bb541b3f77460e4ef986aa6c2771b3d737e75768dd95a18f2b9197c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:52:59.734 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:52:59 smithi046 bash[59995]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b 2024-04-30T09:53:00.161 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:52:59 smithi046 podman[60009]: 2024-04-30 09:52:59.831141569 +0000 UTC m=+0.669198262 container remove 23b6576e6bb541b3f77460e4ef986aa6c2771b3d737e75768dd95a18f2b9197c (image=quay.io/prometheus/node-exporter:v1.3.1, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:53:00.162 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:52:59 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-30T09:53:00.162 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:52:59 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-30T09:53:00.162 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:52:59 smithi046 systemd[1]: Stopped Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:53:00.162 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:52:59 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 5.542s CPU time. 2024-04-30T09:53:00.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:00 smithi026 ceph-mon[22362]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:00 smithi026 ceph-mon[27123]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:00 smithi046 ceph-mon[23181]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:00.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:00 smithi046 systemd[1]: Starting Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:53:00.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:00 smithi046 bash[60140]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-04-30T09:53:01.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:01 smithi046 bash[60140]: Getting image source signatures 2024-04-30T09:53:01.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:01 smithi046 bash[60140]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-04-30T09:53:01.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:01 smithi046 bash[60140]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-04-30T09:53:01.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:01 smithi046 bash[60140]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-04-30T09:53:02.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:02 smithi026 ceph-mon[22362]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:02.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:02 smithi026 ceph-mon[27123]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:02 smithi046 ceph-mon[23181]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:04 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:04.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:04 smithi026 ceph-mon[27123]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:04 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:04.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:04 smithi026 ceph-mon[22362]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:04.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:04 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:04.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:04 smithi046 ceph-mon[23181]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:53:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:53:06] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-30T09:53:07.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:06 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:07.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:06 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:06 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:07.234 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:06 smithi046 bash[60140]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-04-30T09:53:07.234 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 bash[60140]: Writing manifest to image destination 2024-04-30T09:53:07.710 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 podman[60140]: 2024-04-30 09:53:07.40971846 +0000 UTC m=+7.101014098 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-04-30T09:53:07.710 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 podman[60140]: 2024-04-30 09:53:07.516441584 +0000 UTC m=+7.207737256 container create 0a374c1f04f7dc0e96d76f743bc49d3435c6a3d50f1b02c5dba319af2b828f67 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:53:07.972 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-mon[23181]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:07.972 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:07.972 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:07.972 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:07.973 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 podman[60140]: 2024-04-30 09:53:07.710090078 +0000 UTC m=+7.401385705 container init 0a374c1f04f7dc0e96d76f743bc49d3435c6a3d50f1b02c5dba319af2b828f67 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:53:07.973 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 podman[60140]: 2024-04-30 09:53:07.713955851 +0000 UTC m=+7.405251486 container start 0a374c1f04f7dc0e96d76f743bc49d3435c6a3d50f1b02c5dba319af2b828f67 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T09:53:07.973 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.718Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-30T09:53:07.973 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.718Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.719Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.719Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.719Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.719Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=arp 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-30T09:53:07.974 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=edac 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-30T09:53:07.975 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=os 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=stat 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-30T09:53:07.976 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=time 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=uname 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b[60190]: ts=2024-04-30T09:53:07.720Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 bash[60140]: 0a374c1f04f7dc0e96d76f743bc49d3435c6a3d50f1b02c5dba319af2b828f67 2024-04-30T09:53:07.977 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 09:53:07 smithi046 systemd[1]: Started Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:53:08.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:07 smithi026 ceph-mon[22362]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:08.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:07 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:08.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:07 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:08.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:07 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:08.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:07 smithi026 ceph-mon[27123]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:08.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:07 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:08.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:07 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:08.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:07 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:10.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:09 smithi026 ceph-mon[22362]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:10.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:09 smithi026 ceph-mon[27123]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:10.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:09 smithi046 ceph-mon[23181]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:12.027 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:11 smithi026 ceph-mon[27123]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:12.027 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:11 smithi026 ceph-mon[22362]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:11 smithi046 ceph-mon[23181]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:13.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[22362]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:13.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[27123]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:13.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:13 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:13 smithi046 ceph-mon[23181]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:13 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:13 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:13 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:13 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:14 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:14 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:14 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:14 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:14 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:14.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:14 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:15.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:15 smithi026 ceph-mon[22362]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:15.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:15 smithi026 ceph-mon[27123]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:15.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:15 smithi046 ceph-mon[23181]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:16.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:53:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:53:16] "GET /metrics HTTP/1.1" 200 37974 "" "Prometheus/2.33.4" 2024-04-30T09:53:18.152 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:18.152 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.152 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.153 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.153 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.153 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:53:18.153 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:53:18.153 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-30T09:53:18.153 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:18.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:18.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.154 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.155 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.155 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.155 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.155 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.155 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:17 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:18.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:53:18.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:53:18.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-30T09:53:18.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:18.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:18.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 841 B/s rd, 0 op/s 2024-04-30T09:53:18.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:53:18.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:53:18.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-30T09:53:18.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:18.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:18.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:18.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:18.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:19.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:19 smithi026 ceph-mon[22362]: Upgrade: Updating prometheus.a 2024-04-30T09:53:19.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:19 smithi026 ceph-mon[22362]: Deploying daemon prometheus.a on smithi046 2024-04-30T09:53:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:19 smithi026 ceph-mon[22362]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:19 smithi026 ceph-mon[27123]: Upgrade: Updating prometheus.a 2024-04-30T09:53:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:19 smithi026 ceph-mon[27123]: Deploying daemon prometheus.a on smithi046 2024-04-30T09:53:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:19 smithi026 ceph-mon[27123]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:19 smithi046 ceph-mon[23181]: Upgrade: Updating prometheus.a 2024-04-30T09:53:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:19 smithi046 ceph-mon[23181]: Deploying daemon prometheus.a on smithi046 2024-04-30T09:53:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:19 smithi046 ceph-mon[23181]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:22.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:21 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:22.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:21 smithi026 ceph-mon[27123]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 580 B/s rd, 0 op/s 2024-04-30T09:53:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:21 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:21 smithi026 ceph-mon[22362]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 580 B/s rd, 0 op/s 2024-04-30T09:53:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:21 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:21 smithi046 ceph-mon[23181]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 580 B/s rd, 0 op/s 2024-04-30T09:53:22.596 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:53:23.279 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:53:23.279 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (13m) 10s ago 21m 23.0M - 0.23.0 ba2b418f427c 1648d8f22b65 2024-04-30T09:53:23.279 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (13m) 10s ago 21m 52.9M - 8.3.5 dad864ee21e9 9bd49a225988 2024-04-30T09:53:23.279 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 unknown 10s ago 20m 58.5M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:53:23.279 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (44s) 10s ago 26m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:53:23.279 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (10m) 10s ago 28m 502M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:53:23.279 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (28m) 10s ago 28m 77.2M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:53:23.279 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (26m) 10s ago 26m 56.2M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (26m) 10s ago 26m 60.2M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (27s) 10s ago 22m 8028k - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (15s) 10s ago 22m 3330k - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (26m) 10s ago 26m 53.5M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (25m) 10s ago 25m 54.0M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (25m) 10s ago 25m 53.1M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (24m) 10s ago 24m 52.3M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (24m) 10s ago 24m 58.1M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (24m) 10s ago 24m 52.9M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (23m) 10s ago 23m 51.9M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (23m) 10s ago 23m 54.3M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (50s) 10s ago 22m 62.2M - 2.33.4 514e6a882f6e 571af4167709 2024-04-30T09:53:23.280 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (21m) 10s ago 21m 75.6M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:53:23.281 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (21m) 10s ago 21m 75.5M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:53:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:23 smithi026 ceph-mon[27123]: from='client.25444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:23 smithi026 ceph-mon[27123]: from='client.25450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:23 smithi026 ceph-mon[27123]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:23 smithi026 ceph-mon[22362]: from='client.25444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:23 smithi026 ceph-mon[22362]: from='client.25450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:23 smithi026 ceph-mon[22362]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:23 smithi046 ceph-mon[23181]: from='client.25444 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:23 smithi046 ceph-mon[23181]: from='client.25450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:23 smithi046 ceph-mon[23181]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:23.695 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:53:23.695 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:53:23.695 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:53:23.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:53:23.695 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:53:23.695 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:53:23.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:53:23.695 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:53:23.696 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:53:24.042 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading prometheus daemons", 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:53:24.043 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:53:24.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:24 smithi026 ceph-mon[22362]: from='client.25456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:24 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1066976714' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:24 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:24 smithi026 ceph-mon[22362]: from='client.25468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:24 smithi026 ceph-mon[27123]: from='client.25456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:24 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1066976714' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:24 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:24 smithi026 ceph-mon[27123]: from='client.25468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:24 smithi046 ceph-mon[23181]: from='client.25456 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:24 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1066976714' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:24 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:24 smithi046 ceph-mon[23181]: from='client.25468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:25.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:25 smithi026 ceph-mon[22362]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:25.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:25 smithi026 ceph-mon[27123]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:25 smithi046 ceph-mon[23181]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:53:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:53:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:53:26] "GET /metrics HTTP/1.1" 200 37977 "" "Prometheus/2.33.4" 2024-04-30T09:53:27.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:27 smithi026 ceph-mon[27123]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:53:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:27 smithi026 ceph-mon[22362]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:53:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:27 smithi046 ceph-mon[23181]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:53:29.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:29 smithi026 ceph-mon[22362]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:29.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:29 smithi026 ceph-mon[27123]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:29.473 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:29 smithi046 ceph-mon[23181]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:31.078 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:30 smithi046 systemd[1]: Stopping Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:53:31.078 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=main.go:775 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-30T09:53:31.078 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=main.go:798 level=info msg="Stopping scrape discovery manager..." 2024-04-30T09:53:31.078 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=main.go:812 level=info msg="Stopping notify discovery manager..." 2024-04-30T09:53:31.078 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=main.go:834 level=info msg="Stopping scrape manager..." 2024-04-30T09:53:31.078 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=main.go:794 level=info msg="Scrape discovery manager stopped" 2024-04-30T09:53:31.078 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=main.go:808 level=info msg="Notify discovery manager stopped" 2024-04-30T09:53:31.079 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=manager.go:945 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-30T09:53:31.079 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=manager.go:955 level=info component="rule manager" msg="Rule manager stopped" 2024-04-30T09:53:31.079 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=main.go:828 level=info msg="Scrape manager stopped" 2024-04-30T09:53:31.079 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=notifier.go:600 level=info component=notifier msg="Stopping notification manager..." 2024-04-30T09:53:31.079 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=main.go:1054 level=info msg="Notifier manager stopped" 2024-04-30T09:53:31.079 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[58125]: ts=2024-04-30T09:53:31.067Z caller=main.go:1066 level=info msg="See you next time!" 2024-04-30T09:53:31.343 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 podman[61696]: 2024-04-30 09:53:31.077395268 +0000 UTC m=+0.104048416 container died 571af4167709876b57e7a4ad795d499f0edad561d3925d572493cb07fb0a20a7 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:31.701 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 podman[61696]: 2024-04-30 09:53:31.433382786 +0000 UTC m=+0.460035933 container cleanup 571af4167709876b57e7a4ad795d499f0edad561d3925d572493cb07fb0a20a7 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:31.702 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 bash[61696]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a 2024-04-30T09:53:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:31 smithi046 ceph-mon[23181]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:31.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 podman[61710]: 2024-04-30 09:53:31.701159945 +0000 UTC m=+0.621129754 container remove 571af4167709876b57e7a4ad795d499f0edad561d3925d572493cb07fb0a20a7 (image=quay.io/prometheus/prometheus:v2.33.4, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:31.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-30T09:53:31.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 systemd[1]: Stopped Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:53:31.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:31 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service: Consumed 1.345s CPU time. 2024-04-30T09:53:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:31 smithi026 ceph-mon[22362]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:31 smithi026 ceph-mon[27123]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:32.418 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 systemd[1]: Starting Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:53:32.418 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 podman[61841]: 2024-04-30 09:53:32.195830907 +0000 UTC m=+0.019394678 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-04-30T09:53:32.418 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 podman[61841]: 2024-04-30 09:53:32.310731126 +0000 UTC m=+0.134294881 container create eab2d2da881a553dc2d8439f6fdfec6bad6555616417d554aba1d8e11d7961d8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:32.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 podman[61841]: 2024-04-30 09:53:32.53815595 +0000 UTC m=+0.361719707 container init eab2d2da881a553dc2d8439f6fdfec6bad6555616417d554aba1d8e11d7961d8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:32.984 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 podman[61841]: 2024-04-30 09:53:32.540694596 +0000 UTC m=+0.364258356 container start eab2d2da881a553dc2d8439f6fdfec6bad6555616417d554aba1d8e11d7961d8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.564Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.564Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.564Z caller=main.go:570 level=info host_details="(Linux 5.14.0-437.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Apr 9 12:57:02 UTC 2024 x86_64 smithi046 (none))" 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.564Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.564Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.566Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.566Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.567Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.567Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.569Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.569Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.449µs 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.569Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.576Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=6 2024-04-30T09:53:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.591Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=6 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.602Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=6 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.604Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=6 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.606Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=6 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.607Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=5 maxSegment=6 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.608Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=6 maxSegment=6 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.608Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.975µs wal_replay_duration=38.675176ms wbl_replay_duration=101ns total_replay_duration=38.727485ms 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.611Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.611Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.611Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 bash[61841]: eab2d2da881a553dc2d8439f6fdfec6bad6555616417d554aba1d8e11d7961d8 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 systemd[1]: Started Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.633Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=22.302272ms db_storage=800ns remote_storage=1.108µs web_handler=414ns query_engine=583ns scrape=388.996µs scrape_sd=91.979µs notify=17.984µs notify_sd=22.146µs rules=21.423668ms tracing=5.649µs 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.633Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-30T09:53:32.986 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 09:53:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T09:53:32.633Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-30T09:53:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:33 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:33 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:33 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:33 smithi026 ceph-mon[22362]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:33.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:33 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:33.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:33 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:33.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:33 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:33.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:33 smithi026 ceph-mon[27123]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:33.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:33 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:33.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:33 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:33.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:33 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:33.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:33 smithi046 ceph-mon[23181]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:34.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:34 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:34 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:34.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:34 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:35.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:35 smithi026 ceph-mon[27123]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:35.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:35 smithi026 ceph-mon[22362]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:35.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:35 smithi046 ceph-mon[23181]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:37.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:36 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:36 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:36 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:38.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:37 smithi026 ceph-mon[22362]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:37 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:37 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:37 smithi026 ceph-mon[27123]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:37 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:37 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:37 smithi046 ceph-mon[23181]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:53:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:37 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:38.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:37 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:39 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:39 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:39 smithi026 ceph-mon[22362]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:39.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:39 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:39.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:39 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:39.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:39 smithi026 ceph-mon[27123]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:39.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:39 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:39.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:39 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:39.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:39 smithi046 ceph-mon[23181]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:41 smithi026 ceph-mon[27123]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:41 smithi026 ceph-mon[22362]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:41 smithi046 ceph-mon[23181]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:43.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:43.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:43.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:53:43.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:53:43.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T09:53:43.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:43.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:43.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:43 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.135 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:44.135 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:44.135 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:53:44.135 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:53:44.135 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.136 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.137 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:43 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:44.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:44 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:44.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:44 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:44 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:45 smithi026 ceph-mon[27123]: Upgrade: Updating alertmanager.a 2024-04-30T09:53:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:45 smithi026 ceph-mon[27123]: Deploying daemon alertmanager.a on smithi026 2024-04-30T09:53:46.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:45 smithi026 ceph-mon[27123]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:45 smithi026 ceph-mon[22362]: Upgrade: Updating alertmanager.a 2024-04-30T09:53:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:45 smithi026 ceph-mon[22362]: Deploying daemon alertmanager.a on smithi026 2024-04-30T09:53:46.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:45 smithi026 ceph-mon[22362]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:45 smithi046 ceph-mon[23181]: Upgrade: Updating alertmanager.a 2024-04-30T09:53:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:45 smithi046 ceph-mon[23181]: Deploying daemon alertmanager.a on smithi026 2024-04-30T09:53:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:45 smithi046 ceph-mon[23181]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:46.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:53:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:53:46] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.43.0" 2024-04-30T09:53:48.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:47 smithi026 ceph-mon[27123]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 525 B/s rd, 0 op/s 2024-04-30T09:53:48.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:47 smithi026 ceph-mon[22362]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 525 B/s rd, 0 op/s 2024-04-30T09:53:48.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:47 smithi046 ceph-mon[23181]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 525 B/s rd, 0 op/s 2024-04-30T09:53:50.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:49 smithi026 ceph-mon[27123]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:50.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:49 smithi026 ceph-mon[22362]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:50.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:49 smithi046 ceph-mon[23181]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:52.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:51 smithi026 ceph-mon[27123]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:52.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:51 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:52.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:51 smithi026 ceph-mon[22362]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:52.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:51 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:51 smithi046 ceph-mon[23181]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:53:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:51 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:53:52.415 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:52 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:53:52.416 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[56653]: level=info ts=2024-04-30T09:53:52.374Z caller=main.go:557 msg="Received SIGTERM, exiting gracefully..." 2024-04-30T09:53:52.696 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:52 smithi026 podman[75807]: 2024-04-30 09:53:52.415290261 +0000 UTC m=+0.127260016 container died 1648d8f22b65737e1674c98f8752ca1d04dd0dcaaa8155d726c633474c9ba0ef (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:53.185 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:52 smithi026 podman[75807]: 2024-04-30 09:53:52.959568804 +0000 UTC m=+0.671538558 container cleanup 1648d8f22b65737e1674c98f8752ca1d04dd0dcaaa8155d726c633474c9ba0ef (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:53.185 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:52 smithi026 bash[75807]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a 2024-04-30T09:53:53.446 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:53 smithi026 podman[75821]: 2024-04-30 09:53:53.185112237 +0000 UTC m=+0.764587863 container remove 1648d8f22b65737e1674c98f8752ca1d04dd0dcaaa8155d726c633474c9ba0ef (image=quay.io/prometheus/alertmanager:v0.23.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:53.447 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:53 smithi026 podman[75821]: 2024-04-30 09:53:53.260648298 +0000 UTC m=+0.840123921 volume remove 000f990602969dedfb5f80dd884ae561dc0b1a910c03a83889d086ad45a2c1ab 2024-04-30T09:53:53.805 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:53 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-30T09:53:53.805 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:53 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:53:53.805 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:53 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.554s CPU time. 2024-04-30T09:53:53.805 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:53 smithi026 systemd[1]: Starting Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:53:54.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:53 smithi026 ceph-mon[27123]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:53:54.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:53 smithi026 ceph-mon[22362]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:53:54.197 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:53 smithi026 podman[75952]: 2024-04-30 09:53:53.878119813 +0000 UTC m=+0.018650718 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-04-30T09:53:54.197 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 podman[75952]: 2024-04-30 09:53:54.045621033 +0000 UTC m=+0.186151903 volume create 449d833dbb33922c451fd2d8781277253c820843224c36a60c6347b058c57921 2024-04-30T09:53:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:53 smithi046 ceph-mon[23181]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:53:54.415 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:53:54.501 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 podman[75952]: 2024-04-30 09:53:54.198058405 +0000 UTC m=+0.338589284 container create 1485dc9c824f0c2df94281f2691f0a787ae72f13650712d92a4f1bf1e68dc02e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:54.502 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 podman[75952]: 2024-04-30 09:53:54.501351578 +0000 UTC m=+0.641882464 container init 1485dc9c824f0c2df94281f2691f0a787ae72f13650712d92a4f1bf1e68dc02e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:54.935 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 podman[75952]: 2024-04-30 09:53:54.504475418 +0000 UTC m=+0.645006291 container start 1485dc9c824f0c2df94281f2691f0a787ae72f13650712d92a4f1bf1e68dc02e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:53:54.518Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:53:54.518Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:53:54.519Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:53:54.520Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:53:54.544Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:53:54.544Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:53:54.547Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:53:54.548Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 bash[75952]: 1485dc9c824f0c2df94281f2691f0a787ae72f13650712d92a4f1bf1e68dc02e 2024-04-30T09:53:54.936 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:54 smithi026 systemd[1]: Started Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:53:55.124 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:53:55.124 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 starting - - - - 2024-04-30T09:53:55.124 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (14m) 17s ago 22m 53.0M - 8.3.5 dad864ee21e9 9bd49a225988 2024-04-30T09:53:55.124 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 unknown 42s ago 21m 58.5M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:53:55.124 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (76s) 17s ago 27m 451M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (11m) 42s ago 28m 502M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (28m) 42s ago 28m 77.2M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (27m) 17s ago 27m 56.3M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (27m) 42s ago 27m 60.2M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (59s) 42s ago 23m 8028k - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (47s) 17s ago 23m 8442k - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (26m) 42s ago 26m 53.5M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (26m) 42s ago 26m 54.0M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (25m) 42s ago 25m 53.1M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (25m) 42s ago 25m 52.3M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (25m) 17s ago 25m 58.1M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (24m) 17s ago 24m 52.9M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (24m) 17s ago 24m 51.9M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (23m) 17s ago 23m 54.3M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:53:55.125 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (22s) 17s ago 23m 47.0M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:53:55.126 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (21m) 42s ago 21m 75.6M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:53:55.126 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (21m) 17s ago 21m 75.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:53:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:55.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:55.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:54 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:54 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:53:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:54 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:54 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:53:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:54 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:53:55.561 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:53:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:53:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:53:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:53:55.561 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:53:55.562 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:53:55.916 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:53:55.916 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:53:55.916 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:53:55.916 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-30T09:53:55.916 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-30T09:53:55.916 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-04-30T09:53:55.917 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:53:55.917 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-30T09:53:55.917 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading alertmanager daemons", 2024-04-30T09:53:55.917 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:53:55.917 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:53:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:55 smithi026 ceph-mon[27123]: from='client.15576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:55 smithi026 ceph-mon[27123]: from='client.25480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:55 smithi026 ceph-mon[27123]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:56.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:55 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1891288183' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:56.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:55 smithi026 ceph-mon[22362]: from='client.15576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:56.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:55 smithi026 ceph-mon[22362]: from='client.25480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:56.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:55 smithi026 ceph-mon[22362]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:56.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:55 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1891288183' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:55 smithi046 ceph-mon[23181]: from='client.15576 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:55 smithi046 ceph-mon[23181]: from='client.25480 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:55 smithi046 ceph-mon[23181]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:55 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1891288183' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:53:56.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:53:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:53:56] "GET /metrics HTTP/1.1" 200 37976 "" "Prometheus/2.43.0" 2024-04-30T09:53:56.941 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:53:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:53:56.521Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001000866s 2024-04-30T09:53:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:56 smithi026 ceph-mon[27123]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:56 smithi026 ceph-mon[27123]: from='client.25498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:56 smithi026 ceph-mon[22362]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:56 smithi026 ceph-mon[22362]: from='client.25498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:56 smithi046 ceph-mon[23181]: from='client.25486 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:56 smithi046 ceph-mon[23181]: from='client.25498 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:53:58.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:57 smithi026 ceph-mon[27123]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:58.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:57 smithi026 ceph-mon[22362]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:53:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:57 smithi046 ceph-mon[23181]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:00.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[27123]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:00.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[22362]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:53:59 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:59 smithi046 ceph-mon[23181]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:59 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:59 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:59 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:53:59 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:01.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:01 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:01.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:01 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:01.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:01 smithi026 ceph-mon[27123]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:54:01.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:01 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:01.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:01 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:01.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:01 smithi026 ceph-mon[22362]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:54:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:01 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:01 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:01 smithi046 ceph-mon[23181]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T09:54:04.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:03 smithi026 ceph-mon[27123]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:04.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:03 smithi026 ceph-mon[22362]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:03 smithi046 ceph-mon[23181]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:04.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:04] ENGINE Bus STOPPING 2024-04-30T09:54:04.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:54:04.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:04] ENGINE Bus STOPPED 2024-04-30T09:54:04.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:04] ENGINE Bus STARTING 2024-04-30T09:54:04.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:04] ENGINE Serving on http://:::9283 2024-04-30T09:54:04.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:04] ENGINE Bus STARTED 2024-04-30T09:54:04.946 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 09:54:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T09:54:04.523Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002757864s 2024-04-30T09:54:05.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:05.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:05.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:05.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 823 B/s rd, 0 op/s 2024-04-30T09:54:05.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-30T09:54:05.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.062 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.063 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:05 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 823 B/s rd, 0 op/s 2024-04-30T09:54:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-30T09:54:05.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 823 B/s rd, 0 op/s 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:05.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:05.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:05 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:06.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:06 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:54:06.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:06 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-30T09:54:06.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:06 smithi026 ceph-mon[27123]: Health check update: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T09:54:06.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:06 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:06.403 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:06 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:54:06.403 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:06 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-30T09:54:06.403 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:06 smithi026 ceph-mon[22362]: Health check update: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T09:54:06.403 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:06 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:06 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-30T09:54:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:06 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2024-04-30T09:54:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:06 smithi046 ceph-mon[23181]: Health check update: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T09:54:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:06 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:54:06] "GET /metrics HTTP/1.1" 200 37968 "" "Prometheus/2.43.0" 2024-04-30T09:54:07.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:07 smithi026 ceph-mon[27123]: Upgrade: Updating grafana.a 2024-04-30T09:54:07.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:07 smithi026 ceph-mon[27123]: Deploying daemon grafana.a on smithi046 2024-04-30T09:54:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:07 smithi026 ceph-mon[27123]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-30T09:54:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:07 smithi026 ceph-mon[22362]: Upgrade: Updating grafana.a 2024-04-30T09:54:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:07 smithi026 ceph-mon[22362]: Deploying daemon grafana.a on smithi046 2024-04-30T09:54:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:07 smithi026 ceph-mon[22362]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-30T09:54:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:07 smithi046 ceph-mon[23181]: Upgrade: Updating grafana.a 2024-04-30T09:54:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:07 smithi046 ceph-mon[23181]: Deploying daemon grafana.a on smithi046 2024-04-30T09:54:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:07 smithi046 ceph-mon[23181]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1003 B/s rd, 0 op/s 2024-04-30T09:54:09.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:09 smithi026 ceph-mon[27123]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:09.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:09 smithi026 ceph-mon[22362]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:09 smithi046 ceph-mon[23181]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:11.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:11 smithi026 ceph-mon[27123]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:11.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:11 smithi026 ceph-mon[22362]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:11 smithi046 ceph-mon[23181]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:13.420 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:13 smithi046 ceph-mon[23181]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:13 smithi026 ceph-mon[27123]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:13 smithi026 ceph-mon[22362]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:15.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:15 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:15.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:15 smithi046 ceph-mon[23181]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:54:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:15 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:15.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:15 smithi026 ceph-mon[27123]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:54:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:15 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:15 smithi026 ceph-mon[22362]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:54:15.886 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:15 smithi046 systemd[1]: Stopping Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:54:16.204 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[47111]: t=2024-04-30T09:54:15+0000 lvl=info msg="Shutdown started" logger=server reason="System signal: terminated" 2024-04-30T09:54:16.204 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:15 smithi046 podman[63914]: 2024-04-30 09:54:15.895130149 +0000 UTC m=+0.143504372 container died 9bd49a22598807b17d78024541c8dff4cc28e0aa7fc9ee5e88424f66630178b5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, build-date=2022-03-28T10:36:18.413762, distribution-scope=public, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, version=8.5, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, description=Ceph Grafana Container, release=236.1648460182, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.buildah.version=1.24.2, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-04-30T09:54:16.484 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:16 smithi046 podman[63914]: 2024-04-30 09:54:16.368022126 +0000 UTC m=+0.616396354 container cleanup 9bd49a22598807b17d78024541c8dff4cc28e0aa7fc9ee5e88424f66630178b5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, description=Ceph Grafana Container, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-03-28T10:36:18.413762, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, architecture=x86_64, distribution-scope=public, release=236.1648460182, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, io.openshift.expose-services=, io.buildah.version=1.24.2, name=ubi8) 2024-04-30T09:54:16.484 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:16 smithi046 bash[63914]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a 2024-04-30T09:54:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:54:16] "GET /metrics HTTP/1.1" 200 37968 "" "Prometheus/2.43.0" 2024-04-30T09:54:16.934 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:16 smithi046 podman[63928]: 2024-04-30 09:54:16.586013116 +0000 UTC m=+0.692170735 container remove 9bd49a22598807b17d78024541c8dff4cc28e0aa7fc9ee5e88424f66630178b5 (image=quay.io/ceph/ceph-grafana:8.3.5, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.openshift.expose-services=, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, vcs-type=git, io.buildah.version=1.24.2, description=Ceph Grafana Container, build-date=2022-03-28T10:36:18.413762, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=236.1648460182, version=8.5, architecture=x86_64, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-236.1648460182, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc.) 2024-04-30T09:54:16.934 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:16 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-30T09:54:16.934 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:16 smithi046 systemd[1]: Stopped Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:54:16.934 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:16 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@grafana.a.service: Consumed 5.720s CPU time. 2024-04-30T09:54:17.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:16 smithi026 ceph-mon[22362]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:17.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:16 smithi026 ceph-mon[27123]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:16 smithi046 ceph-mon[23181]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:17.234 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:16 smithi046 systemd[1]: Starting Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:54:17.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 podman[64070]: 2024-04-30 09:54:17.055766863 +0000 UTC m=+0.019188030 image pull 2bacad6d85d802cd176c146528adcc0e47a8a83e94bc4a8b7ba1a9b9ef2c65e7 quay.io/ceph/grafana:9.4.12 2024-04-30T09:54:17.235 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 podman[64070]: 2024-04-30 09:54:17.170891245 +0000 UTC m=+0.134312443 container create a88d0112f71cb0e765405764b978160c3055be65b7bbde5c707937d8b8d772d7 (image=quay.io/ceph/grafana:9.4.12, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-30T09:54:17.526 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 podman[64070]: 2024-04-30 09:54:17.356425129 +0000 UTC m=+0.319846282 container init a88d0112f71cb0e765405764b978160c3055be65b7bbde5c707937d8b8d772d7 (image=quay.io/ceph/grafana:9.4.12, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-30T09:54:17.526 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 podman[64070]: 2024-04-30 09:54:17.374059934 +0000 UTC m=+0.337481087 container start a88d0112f71cb0e765405764b978160c3055be65b7bbde5c707937d8b8d772d7 (image=quay.io/ceph/grafana:9.4.12, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-30T09:54:17.526 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 bash[64070]: a88d0112f71cb0e765405764b978160c3055be65b7bbde5c707937d8b8d772d7 2024-04-30T09:54:17.526 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 systemd[1]: Started Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:54:17.985 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.52552061Z level=info msg="Starting Grafana" version=9.4.12 commit=37ee25ba46 branch=HEAD compiled=2023-05-22T15:58:05Z 2024-04-30T09:54:17.985 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525701289Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-30T09:54:17.985 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525760908Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-30T09:54:17.985 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525771319Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-30T09:54:17.985 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525776397Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-04-30T09:54:17.985 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525782162Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-04-30T09:54:17.985 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525786756Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525791841Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525796259Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525802227Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525807306Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525812338Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525817218Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.52582325Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525827786Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525831765Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525835849Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-30T09:54:17.986 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525840088Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-30T09:54:17.987 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=settings t=2024-04-30T09:54:17.525845208Z level=info msg="App mode production" 2024-04-30T09:54:17.987 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=sqlstore t=2024-04-30T09:54:17.525904478Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-30T09:54:17.987 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=sqlstore t=2024-04-30T09:54:17.52592264Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-30T09:54:17.987 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.53778173Z level=info msg="Starting DB migrations" 2024-04-30T09:54:17.988 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.541444416Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-30T09:54:17.988 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.553659772Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-30T09:54:17.988 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.55477803Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-30T09:54:17.988 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.555083088Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-30T09:54:17.988 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.556285086Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-30T09:54:17.988 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.557365321Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-30T09:54:17.988 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.558404607Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-30T09:54:17.988 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.55868293Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.559172304Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.559709605Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.560069876Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.561200567Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.561520036Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.562641022Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.563742332Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.564790126Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.565072258Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-30T09:54:17.990 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.566142696Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.567201949Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.567478304Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.568553299Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.569614049Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.570062763Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.570489407Z level=info msg="Executing migration" id="create alert_image table" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.570922222Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.571356493Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.57159832Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.572018661Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.572401589Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.572477909Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.572693587Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.573153934Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-30T09:54:17.991 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.573541029Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.573847269Z level=info msg="Executing migration" id="create secrets table" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.574351738Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.580691737Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.5818295Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.582119153Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.589475593Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.596102433Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.597221878Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.597612259Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.598137343Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.598377908Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.598799516Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.60019418Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.600923654Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.601222548Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-30T09:54:17.992 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.601549946Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-30T09:54:17.993 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.601978357Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-30T09:54:17.993 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.602416288Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-30T09:54:17.993 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.60355448Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.603870782Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.604332001Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.604886247Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.605310534Z level=info msg="Executing migration" id="add correlation config column" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.606442514Z level=info msg="Executing migration" id="create entity_events table" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.606877727Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.60728112Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.607360843Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.607586273Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.607681183Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.60797768Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.608386815Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-30T09:54:17.994 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.608812376Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.609423252Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.609897018Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.610300784Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.610701032Z level=info msg="Executing migration" id="Drop public config table" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.611111533Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.61148619Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.611922665Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.612337403Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.612820312Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.616652906Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.618064272Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.619487267Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-30T09:54:17.995 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.619916764Z level=info msg="Executing migration" id="add share column" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.621156787Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.621683754Z level=info msg="Executing migration" id="create file table" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.62214087Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.622575485Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.623026584Z level=info msg="Executing migration" id="create file_meta table" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.623402728Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.623866431Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.624132128Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.625245522Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.625892105Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.626533154Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.627704558Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-30T09:54:17.996 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.627993603Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.628416819Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.628821504Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.629302312Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.629623988Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.630793517Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.631945339Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.63235561Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.646588052Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.647063523Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.647494017Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.651987768Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.652488126Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.652827226Z level=info msg="Executing migration" id="create folder table" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.653205173Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.653624194Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-30T09:54:17.997 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.654089204Z level=info msg="Executing migration" id="Update folder title length" 2024-04-30T09:54:17.998 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.654322301Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-30T09:54:17.998 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=migrator t=2024-04-30T09:54:17.654789775Z level=info msg="migrations completed" performed=105 skipped=376 duration=113.446925ms 2024-04-30T09:54:17.998 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=sqlstore t=2024-04-30T09:54:17.655150874Z level=info msg="Created default organization" 2024-04-30T09:54:17.998 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=plugin.finder t=2024-04-30T09:54:17.689402854Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-04-30T09:54:17.998 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=secrets t=2024-04-30T09:54:17.689497732Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-30T09:54:17.998 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=query_data t=2024-04-30T09:54:17.691705319Z level=info msg="Query Service initialization" 2024-04-30T09:54:17.998 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=live.push_http t=2024-04-30T09:54:17.694429004Z level=info msg="Live Push Gateway initialization" 2024-04-30T09:54:18.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:18.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:18.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:18.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-mon[23181]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=infra.usagestats.collector t=2024-04-30T09:54:18.391858873Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=provisioning.datasources t=2024-04-30T09:54:18.393729124Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=provisioning.datasources t=2024-04-30T09:54:18.393738601Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=provisioning.datasources t=2024-04-30T09:54:18.393848947Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=provisioning.alerting t=2024-04-30T09:54:18.41636288Z level=info msg="starting to provision alerting" 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=provisioning.alerting t=2024-04-30T09:54:18.416377011Z level=info msg="finished to provision alerting" 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=grafanaStorageLogger t=2024-04-30T09:54:18.416591711Z level=info msg="storage starting" 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=ngalert.state.manager t=2024-04-30T09:54:18.416676713Z level=info msg="Warming state cache for startup" 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=http.server t=2024-04-30T09:54:18.419107495Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=ngalert.state.manager t=2024-04-30T09:54:18.420864959Z level=info msg="State cache has been initialized" states=0 duration=4.186929ms 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=ticker t=2024-04-30T09:54:18.420906627Z level=info msg=starting first_tick=2024-04-30T09:54:20Z 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=ngalert.multiorg.alertmanager t=2024-04-30T09:54:18.420918946Z level=info msg="starting MultiOrg Alertmanager" 2024-04-30T09:54:18.646 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=sqlstore.transactions t=2024-04-30T09:54:18.432608095Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-30T09:54:18.647 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=sqlstore.transactions t=2024-04-30T09:54:18.443665638Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-30T09:54:18.647 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 09:54:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=sqlstore.transactions t=2024-04-30T09:54:18.454370295Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-04-30T09:54:18.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:18 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:18.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:18 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:18.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:18 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:18.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:18 smithi026 ceph-mon[27123]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:18.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:18 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:18.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:18 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:18.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:18 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:18.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:18 smithi026 ceph-mon[22362]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:21 smithi046 ceph-mon[23181]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:21 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:21.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:21 smithi026 ceph-mon[22362]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:21.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:21 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:21 smithi026 ceph-mon[27123]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:21.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:21 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:23.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:23 smithi026 ceph-mon[22362]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:23 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:23 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:23.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:23 smithi026 ceph-mon[27123]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:23.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:23.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:23.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:23 smithi046 ceph-mon[23181]: pgmap v69: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:23.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:23 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:23.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:23 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:24 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:24 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:24 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:24 smithi046 ceph-mon[23181]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:24.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:24 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:24.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:24 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:24.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:24 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:24.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:24 smithi026 ceph-mon[22362]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:24 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:24 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:24 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:24 smithi026 ceph-mon[27123]: pgmap v70: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:26.295 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:54:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:54:26] "GET /metrics HTTP/1.1" 200 37973 "" "Prometheus/2.43.0" 2024-04-30T09:54:26.975 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:54:26.975 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (33s) 27s ago 22m 15.7M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:54:26.975 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (9s) 4s ago 22m 53.5M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:54:26.975 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 27s ago 21m 61.0M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (108s) 4s ago 27m 451M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (12m) 27s ago 29m 504M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (29m) 27s ago 29m 79.4M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (27m) 4s ago 27m 56.3M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (27m) 27s ago 27m 60.9M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (90s) 27s ago 23m 8468k - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (79s) 4s ago 23m 8250k - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (27m) 27s ago 27m 53.5M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (26m) 27s ago 26m 54.0M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (26m) 27s ago 26m 53.1M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (26m) 27s ago 25m 52.3M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (25m) 4s ago 25m 58.0M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (25m) 4s ago 25m 52.9M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (24m) 4s ago 24m 51.9M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (24m) 4s ago 24m 54.3M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:54:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (54s) 4s ago 23m 48.7M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:54:26.977 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (22m) 27s ago 22m 75.8M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:54:26.977 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (22m) 4s ago 22m 75.7M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:54:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:26 smithi026 ceph-mon[27123]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:27.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:26 smithi026 ceph-mon[22362]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:26 smithi046 ceph-mon[23181]: pgmap v71: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:54:27.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:54:27.389 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:54:27.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:54:27.389 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:54:27.389 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-30T09:54:27.389 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:54:27.389 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:54:27.389 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:54:27.737 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:54:27.737 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:54:27.737 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:54:27.737 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-04-30T09:54:27.738 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-30T09:54:27.738 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr" 2024-04-30T09:54:27.738 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:54:27.738 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-30T09:54:27.738 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading grafana daemons", 2024-04-30T09:54:27.738 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:54:27.738 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:54:28.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:27 smithi026 ceph-mon[27123]: from='client.25504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:27 smithi026 ceph-mon[27123]: from='client.25510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:27 smithi026 ceph-mon[27123]: from='client.25516 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:28.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:27 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4013656267' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:27 smithi026 ceph-mon[22362]: from='client.25504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:27 smithi026 ceph-mon[22362]: from='client.25510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:27 smithi026 ceph-mon[22362]: from='client.25516 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:28.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:27 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4013656267' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:27 smithi046 ceph-mon[23181]: from='client.25504 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:27 smithi046 ceph-mon[23181]: from='client.25510 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:27 smithi046 ceph-mon[23181]: from='client.25516 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:28.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:27 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4013656267' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:28 smithi026 ceph-mon[27123]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:29.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:28 smithi026 ceph-mon[27123]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:28 smithi026 ceph-mon[22362]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:29.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:28 smithi026 ceph-mon[22362]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:28 smithi046 ceph-mon[23181]: from='client.15630 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:28 smithi046 ceph-mon[23181]: pgmap v72: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:30.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:29] ENGINE Bus STOPPING 2024-04-30T09:54:30.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:54:30.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:29] ENGINE Bus STOPPED 2024-04-30T09:54:30.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:29] ENGINE Bus STARTING 2024-04-30T09:54:30.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:29] ENGINE Serving on http://:::9283 2024-04-30T09:54:30.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:54:29] ENGINE Bus STARTED 2024-04-30T09:54:30.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 877 B/s rd, 0 op/s 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046.front.sepia.ceph.com:3000"}]: dispatch 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046.front.sepia.ceph.com:3000"}]: dispatch 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:30.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: Upgrade: Finalizing container_image settings 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T09:54:30.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 877 B/s rd, 0 op/s 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046.front.sepia.ceph.com:3000"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046.front.sepia.ceph.com:3000"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: Upgrade: Finalizing container_image settings 2024-04-30T09:54:30.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: Upgrade: Complete! 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:54:30.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 647 B/s rd, 0 op/s 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:54:30.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:54:30.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:54:30.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T09:54:30.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:54:30.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:54:30.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:30.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:30.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:54:30.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: Upgrade: Complete! 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 647 B/s rd, 0 op/s 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:30.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:30.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:30.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:54:30.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:30 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: pgmap v73: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 877 B/s rd, 0 op/s 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: pgmap v74: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046.front.sepia.ceph.com:3000"}]: dispatch 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi046.front.sepia.ceph.com:3000"}]: dispatch 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: Upgrade: Finalizing container_image settings 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T09:54:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: Upgrade: Complete! 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:30.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: pgmap v75: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 647 B/s rd, 0 op/s 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: pgmap v76: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 866 B/s rd, 0 op/s 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: pgmap v77: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: pgmap v78: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:54:30.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:30 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:32.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:32 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:32.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:32 smithi026 ceph-mon[27123]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-04-30T09:54:32.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:32 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:32.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:32 smithi026 ceph-mon[22362]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-04-30T09:54:32.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:32 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:54:32.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:32 smithi046 ceph-mon[23181]: pgmap v79: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-04-30T09:54:34.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:34 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:34.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:34 smithi026 ceph-mon[27123]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:34 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:34 smithi026 ceph-mon[22362]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:34 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:34 smithi046 ceph-mon[23181]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:54:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:54:36] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.43.0" 2024-04-30T09:54:37.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:36 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:37.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:36 smithi026 ceph-mon[22362]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-04-30T09:54:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:36 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:36 smithi026 ceph-mon[27123]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-04-30T09:54:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:36 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:36 smithi046 ceph-mon[23181]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-04-30T09:54:38.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:38 smithi026 ceph-mon[27123]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:38 smithi026 ceph-mon[22362]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:38 smithi046 ceph-mon[23181]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:40.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:40 smithi026 ceph-mon[27123]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:54:40.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:40 smithi026 ceph-mon[22362]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:54:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:40 smithi046 ceph-mon[23181]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:54:42.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:42 smithi026 ceph-mon[22362]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:42 smithi026 ceph-mon[27123]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:42.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:42 smithi046 ceph-mon[23181]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:44.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:44 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:44 smithi026 ceph-mon[27123]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:44 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:44 smithi026 ceph-mon[22362]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:44 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:44 smithi046 ceph-mon[23181]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:46.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:54:46] "GET /metrics HTTP/1.1" 200 37986 "" "Prometheus/2.43.0" 2024-04-30T09:54:47.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:46 smithi026 ceph-mon[27123]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:47.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:46 smithi026 ceph-mon[22362]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:47.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:46 smithi046 ceph-mon[23181]: pgmap v86: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:48 smithi026 ceph-mon[22362]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:48.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:48 smithi026 ceph-mon[27123]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:48 smithi046 ceph-mon[23181]: pgmap v87: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:50 smithi026 ceph-mon[27123]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:50 smithi026 ceph-mon[22362]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:50 smithi046 ceph-mon[23181]: pgmap v88: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:52.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:51 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:52.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:51 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:51 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:54:53.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:52 smithi026 ceph-mon[27123]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:53.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:52 smithi026 ceph-mon[22362]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:53.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:52 smithi046 ceph-mon[23181]: pgmap v89: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:54.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:54 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:54.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:54 smithi026 ceph-mon[27123]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:54 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:54 smithi026 ceph-mon[22362]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:54 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:54:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:54 smithi046 ceph-mon[23181]: pgmap v90: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:56.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:54:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:54:56] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.43.0" 2024-04-30T09:54:57.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:56 smithi026 ceph-mon[27123]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:57.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:56 smithi026 ceph-mon[22362]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:57.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:56 smithi046 ceph-mon[23181]: pgmap v91: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:54:58.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:58 smithi026 ceph-mon[27123]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:58.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:54:58 smithi026 ceph-mon[27123]: from='client.25534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:58.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:58 smithi026 ceph-mon[22362]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:58.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:54:58 smithi026 ceph-mon[22362]: from='client.25534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:58 smithi046 ceph-mon[23181]: pgmap v92: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:54:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:54:58 smithi046 ceph-mon[23181]: from='client.25534 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:54:58.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.mgr | length == 1'"'"'' 2024-04-30T09:55:00.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:00 smithi026 ceph-mon[27123]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:00 smithi026 ceph-mon[22362]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:00 smithi046 ceph-mon[23181]: pgmap v93: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:00.731 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:55:01.321 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:01 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/1136115336' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:01.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:01 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/1136115336' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:01.346 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.mgr | keys'"'"' | grep $sha1' 2024-04-30T09:55:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:01 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/1136115336' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:02.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:02 smithi026 ceph-mon[27123]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:02.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:02 smithi026 ceph-mon[22362]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:02 smithi046 ceph-mon[23181]: pgmap v94: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:03.446 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)" 2024-04-30T09:55:03.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:03 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/4115950888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:03.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:03 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/4115950888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:03.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:03 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/4115950888' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:04.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 2'"'"'' 2024-04-30T09:55:04.689 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:04 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:04.689 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:04 smithi026 ceph-mon[22362]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:04.689 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:04 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:04.689 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:04 smithi026 ceph-mon[27123]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:04.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:04 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:04.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:04 smithi046 ceph-mon[23181]: pgmap v95: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:06.155 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:55:06.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:55:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:55:06] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.43.0" 2024-04-30T09:55:06.811 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 2'"'"'' 2024-04-30T09:55:07.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:06 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:07.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:06 smithi026 ceph-mon[22362]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:07.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:06 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/218806039' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:07.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:06 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:07.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:06 smithi026 ceph-mon[27123]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:07.128 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:06 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/218806039' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:06 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:06 smithi046 ceph-mon[23181]: pgmap v96: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:06 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/218806039' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:08.305 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:08 smithi026 ceph-mon[27123]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:08.306 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:08 smithi026 ceph-mon[22362]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:08.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:08 smithi046 ceph-mon[23181]: pgmap v97: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:09.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:09 smithi026 ceph-mon[27123]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:09.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:09 smithi026 ceph-mon[22362]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:09 smithi046 ceph-mon[23181]: from='client.25558 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:10.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:10 smithi026 ceph-mon[27123]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:10.316 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:10 smithi026 ceph-mon[22362]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:10 smithi046 ceph-mon[23181]: pgmap v98: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:11.233 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:55:11.866 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade status' 2024-04-30T09:55:12.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:12 smithi026 ceph-mon[27123]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:12 smithi026 ceph-mon[22362]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:12 smithi046 ceph-mon[23181]: pgmap v99: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:13.919 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:55:13.919 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-30T09:55:13.919 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-30T09:55:13.919 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-30T09:55:13.919 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:55:13.919 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-30T09:55:13.919 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-30T09:55:13.919 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:55:13.919 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:55:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:14 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:14 smithi026 ceph-mon[27123]: from='client.15666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:14 smithi026 ceph-mon[27123]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:14 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:14 smithi026 ceph-mon[22362]: from='client.15666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:14 smithi026 ceph-mon[22362]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:14 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:14 smithi046 ceph-mon[23181]: from='client.15666 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:14 smithi046 ceph-mon[23181]: pgmap v100: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:14.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T09:55:16.517 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:55:16.517 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:55:16.517 INFO:teuthology.orchestra.run.smithi026.stdout: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:55:16.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:55:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:55:16] "GET /metrics HTTP/1.1" 200 37982 "" "Prometheus/2.43.0" 2024-04-30T09:55:17.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:16 smithi026 ceph-mon[27123]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:17.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:16 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/2305056403' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:55:17.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:16 smithi026 ceph-mon[22362]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:17.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:16 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/2305056403' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:55:17.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '"'"'{print $2}'"'"')' 2024-04-30T09:55:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:16 smithi046 ceph-mon[23181]: pgmap v101: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:16 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/2305056403' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:55:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:18 smithi026 ceph-mon[27123]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:18 smithi026 ceph-mon[22362]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:18 smithi046 ceph-mon[23181]: pgmap v102: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:20.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:20 smithi026 ceph-mon[27123]: from='client.25576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:20.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:20 smithi026 ceph-mon[27123]: from='client.25582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "mon", "hosts": "smithi046", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:20 smithi026 ceph-mon[27123]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:20 smithi026 ceph-mon[22362]: from='client.25576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:20 smithi026 ceph-mon[22362]: from='client.25582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "mon", "hosts": "smithi046", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:20 smithi026 ceph-mon[22362]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:20 smithi046 ceph-mon[23181]: from='client.25576 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:20 smithi046 ceph-mon[23181]: from='client.25582 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "mon", "hosts": "smithi046", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:20 smithi046 ceph-mon[23181]: pgmap v103: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:22.027 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:55:22.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:21 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:22.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:21 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:22.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:21 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:22.662 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-30T09:55:23.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[27123]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:55:23.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:23.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:23.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[27123]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:23.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:23.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:23.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[22362]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:55:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[22362]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:23 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:23 smithi046 ceph-mon[23181]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:55:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:23 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:23 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:23 smithi046 ceph-mon[23181]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:23 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:23 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:23 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:24.142 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:24 smithi026 ceph-mon[27123]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-04-30T09:55:24.142 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:24 smithi026 ceph-mon[27123]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:24.142 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:24 smithi026 ceph-mon[27123]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:55:24.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:24 smithi026 ceph-mon[22362]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-04-30T09:55:24.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:24 smithi026 ceph-mon[22362]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:24.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:24 smithi026 ceph-mon[22362]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:55:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:24 smithi046 ceph-mon[23181]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 949 B/s rd, 0 op/s 2024-04-30T09:55:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:24 smithi046 ceph-mon[23181]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:24 smithi046 ceph-mon[23181]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:55:24.640 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:55:25.270 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:25 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:25.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:25 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:25.334 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:55:25.334 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (91s) 85s ago 23m 15.7M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:55:25.334 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (68s) 63s ago 23m 53.5M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:55:25.334 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 85s ago 22m 61.0M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:55:25.334 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (2m) 63s ago 28m 451M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (12m) 85s ago 30m 504M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (30m) 85s ago 30m 79.4M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (28m) 63s ago 28m 56.3M 2048M 17.2.0 e1d6a67b021e e7247b5c791f 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (28m) 85s ago 28m 60.9M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) 85s ago 24m 8468k - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (2m) 63s ago 24m 8250k - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (28m) 85s ago 28m 53.5M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (27m) 85s ago 27m 54.0M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (27m) 85s ago 27m 53.1M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (26m) 85s ago 26m 52.3M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (26m) 63s ago 26m 58.0M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (26m) 63s ago 26m 52.9M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (25m) 63s ago 25m 51.9M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (25m) 63s ago 25m 54.3M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:55:25.335 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (113s) 63s ago 24m 48.7M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:55:25.336 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (23m) 85s ago 23m 75.8M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:55:25.336 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (23m) 63s ago 23m 75.7M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:55:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:25 smithi046 ceph-mon[23181]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:25.761 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:55:25.761 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:55:25.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:55:25.763 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:55:25.763 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 13, 2024-04-30T09:55:25.763 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:55:25.763 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:55:25.763 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:55:26.114 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:55:26.114 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", 2024-04-30T09:55:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:55:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi046", 2024-04-30T09:55:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:55:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "", 2024-04-30T09:55:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 image", 2024-04-30T09:55:26.115 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:55:26.115 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:55:26.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:26 smithi026 ceph-mon[27123]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:26.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:26 smithi026 ceph-mon[27123]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:26.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:26 smithi026 ceph-mon[27123]: from='client.25594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:26.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:26 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3143229326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:26.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:26 smithi026 ceph-mon[22362]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:26.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:26 smithi026 ceph-mon[22362]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:26.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:26 smithi026 ceph-mon[22362]: from='client.25594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:26.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:26 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3143229326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:26 smithi046 ceph-mon[23181]: from='client.15690 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:26 smithi046 ceph-mon[23181]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:26 smithi046 ceph-mon[23181]: from='client.25594 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:26 smithi046 ceph-mon[23181]: from='client.? 172.21.15.26:0/3143229326' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:55:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:55:26] "GET /metrics HTTP/1.1" 200 37985 "" "Prometheus/2.43.0" 2024-04-30T09:55:27.207 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:27 smithi046 ceph-mon[23181]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:27.207 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:27 smithi046 ceph-mon[23181]: from='client.25612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:27.207 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:27.207 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:27.207 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:27.207 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:27.207 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-30T09:55:27.207 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:27 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-30T09:55:27.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[22362]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:27.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[22362]: from='client.25612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[27123]: from='client.15702 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[27123]: from='client.25612 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-30T09:55:27.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:27 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-30T09:55:28.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:28 smithi046 ceph-mon[23181]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:55:28.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:28 smithi046 ceph-mon[23181]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:55:28.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:28 smithi046 ceph-mon[23181]: Upgrade: Setting container_image for all mgr 2024-04-30T09:55:28.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:28 smithi046 ceph-mon[23181]: Upgrade: It appears safe to stop mon.b 2024-04-30T09:55:28.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:28 smithi046 ceph-mon[23181]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-30T09:55:28.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:28 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:28.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:28 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:55:28.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:28 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:55:28.422 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:28 smithi046 ceph-mon[23181]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:28.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[22362]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[22362]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all mgr 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[22362]: Upgrade: It appears safe to stop mon.b 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[22362]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[27123]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[27123]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all mgr 2024-04-30T09:55:28.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[27123]: Upgrade: It appears safe to stop mon.b 2024-04-30T09:55:28.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[27123]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-30T09:55:28.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:28.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:55:28.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:55:28.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:28 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:29.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:29 smithi026 ceph-mon[27123]: Upgrade: Updating mon.b 2024-04-30T09:55:29.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:29 smithi026 ceph-mon[27123]: Deploying daemon mon.b on smithi046 2024-04-30T09:55:29.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:29 smithi026 ceph-mon[22362]: Upgrade: Updating mon.b 2024-04-30T09:55:29.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:29 smithi026 ceph-mon[22362]: Deploying daemon mon.b on smithi046 2024-04-30T09:55:29.475 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:29 smithi046 ceph-mon[23181]: Upgrade: Updating mon.b 2024-04-30T09:55:29.475 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:29 smithi046 ceph-mon[23181]: Deploying daemon mon.b on smithi046 2024-04-30T09:55:30.031 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:29 smithi046 systemd[1]: Stopping Ceph mon.b for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:55:30.480 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:30 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b[23175]: 2024-04-30T09:55:30.029+0000 7f28c2988700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T09:55:30.481 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:30 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b[23175]: 2024-04-30T09:55:30.029+0000 7f28c2988700 -1 mon.b@2(peon) e3 *** Got Signal Terminated *** 2024-04-30T09:55:30.481 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:30 smithi046 podman[65524]: 2024-04-30 09:55:30.101014697 +0000 UTC m=+0.190805781 container died e7247b5c791f3583e5b0596c03aa606e2a00412f99e17784a1d6a833bab2c4f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, vcs-type=git, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, architecture=x86_64, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.openshift.tags=base centos centos-stream, ceph=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8) 2024-04-30T09:55:30.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:30 smithi046 podman[65524]: 2024-04-30 09:55:30.578969435 +0000 UTC m=+0.668760516 container cleanup e7247b5c791f3583e5b0596c03aa606e2a00412f99e17784a1d6a833bab2c4f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_BRANCH=HEAD, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.component=centos-stream-container, ceph=True, maintainer=Guillaume Abrioux , architecture=x86_64, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, io.k8s.display-name=CentOS Stream 8, release=754, GIT_CLEAN=True, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-30T09:55:30.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:30 smithi046 bash[65524]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b 2024-04-30T09:55:31.163 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:30 smithi046 podman[65538]: 2024-04-30 09:55:30.838360129 +0000 UTC m=+0.735484029 container remove e7247b5c791f3583e5b0596c03aa606e2a00412f99e17784a1d6a833bab2c4f7 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, GIT_CLEAN=True, com.redhat.component=centos-stream-container, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vendor=Red Hat, Inc., io.openshift.expose-services=, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, distribution-scope=public, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, architecture=x86_64) 2024-04-30T09:55:31.163 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:30 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-30T09:55:31.163 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:30 smithi046 systemd[1]: Stopped Ceph mon.b for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:55:31.163 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:30 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.b.service: Consumed 19.267s CPU time. 2024-04-30T09:55:31.474 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 systemd[1]: Starting Ceph mon.b for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:55:31.474 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 podman[65670]: 2024-04-30 09:55:31.373102988 +0000 UTC m=+0.107770758 container create 9bfb997f7d880f28d1c60241527360159d2b018853689f4034a8e63c04290d99 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-04-30T09:55:31.474 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 podman[65670]: 2024-04-30 09:55:31.284116688 +0000 UTC m=+0.018784444 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:55:31.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 podman[65670]: 2024-04-30 09:55:31.550760513 +0000 UTC m=+0.285428242 container init 9bfb997f7d880f28d1c60241527360159d2b018853689f4034a8e63c04290d99 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-04-30T09:55:31.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 podman[65670]: 2024-04-30 09:55:31.554272461 +0000 UTC m=+0.288940197 container start 9bfb997f7d880f28d1c60241527360159d2b018853689f4034a8e63c04290d99 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-30T09:55:31.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: set uid:gid to 167:167 (ceph:ceph) 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev), process ceph-mon, pid 2 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: pidfile_write: ignore empty --pid-file 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: load: jerasure load: lrc 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: RocksDB version: 7.9.2 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Git sha 0 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Compile date 2024-04-30 00:29:27 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: DB SUMMARY 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: DB Session ID: CFI5QDN5DW1EBTYN5JAC 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: CURRENT file: CURRENT 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: IDENTITY file: IDENTITY 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3050 Bytes 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000057.sst 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000055.log size: 359813 ; 2024-04-30T09:55:31.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.error_if_exists: 0 2024-04-30T09:55:31.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.create_if_missing: 0 2024-04-30T09:55:31.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.paranoid_checks: 1 2024-04-30T09:55:31.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-30T09:55:31.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-30T09:55:31.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-30T09:55:31.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.env: 0x55d5f6136cc0 2024-04-30T09:55:31.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.fs: PosixFileSystem 2024-04-30T09:55:31.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.info_log: 0x55d5f6a8fda0 2024-04-30T09:55:31.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_file_opening_threads: 16 2024-04-30T09:55:31.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.statistics: (nil) 2024-04-30T09:55:31.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.use_fsync: 0 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_log_file_size: 0 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.keep_log_file_num: 1000 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.recycle_log_file_num: 0 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.allow_fallocate: 1 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.allow_mmap_reads: 0 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.allow_mmap_writes: 0 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.use_direct_reads: 0 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.create_missing_column_families: 0 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.db_log_dir: 2024-04-30T09:55:31.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.wal_dir: 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.advise_random_on_open: 1 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.db_write_buffer_size: 0 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.write_buffer_manager: 0x55d5f6a610e0 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.rate_limiter: (nil) 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-30T09:55:31.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.wal_recovery_mode: 2 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.enable_thread_tracking: 0 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.enable_pipelined_write: 0 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.unordered_write: 0 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.row_cache: None 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.wal_filter: None 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.allow_ingest_behind: 0 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.two_write_queues: 0 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.manual_wal_flush: 0 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.wal_compression: 0 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.atomic_flush: 0 2024-04-30T09:55:31.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.log_readahead_size: 0 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.best_efforts_recovery: 0 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.allow_data_in_errors: 0 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.db_host_id: __hostname__ 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_background_jobs: 2 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_background_compactions: -1 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_subcompactions: 1 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-30T09:55:31.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_total_wal_size: 0 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_open_files: -1 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bytes_per_sync: 0 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_readahead_size: 0 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_background_flushes: -1 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Compression algorithms supported: 2024-04-30T09:55:31.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: kZSTD supported: 0 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: kXpressCompression supported: 0 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: kLZ4HCCompression supported: 1 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: kZlibCompression supported: 1 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: kSnappyCompression supported: 1 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: kLZ4Compression supported: 1 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: kBZip2Compression supported: 0 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.merge_operator: 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_filter: None 2024-04-30T09:55:31.743 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_filter_factory: None 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.sst_partitioner_factory: None 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d5f6a8ff60) 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: cache_index_and_filter_blocks: 1 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: pin_top_level_index_and_filter: 1 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: index_type: 0 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: data_block_index_type: 0 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: index_shortening: 1 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: checksum: 4 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: no_block_cache: 0 2024-04-30T09:55:31.744 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_cache: 0x55d5f6a931f0 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_cache_name: BinnedLRUCache 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_cache_options: 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: capacity : 536870912 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: num_shard_bits : 4 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: strict_capacity_limit : 0 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: high_pri_pool_ratio: 0.000 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_cache_compressed: (nil) 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: persistent_cache: (nil) 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_size: 4096 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_size_deviation: 10 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_restart_interval: 16 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: index_block_restart_interval: 1 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: metadata_block_size: 4096 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: partition_filters: 0 2024-04-30T09:55:31.745 INFO:journalctl@ceph.mon.b.smithi046.stdout: use_delta_encoding: 1 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: filter_policy: bloomfilter 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: whole_key_filtering: 1 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: verify_compression: 0 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: read_amp_bytes_per_bit: 0 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: format_version: 5 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: enable_index_compression: 1 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: block_align: 0 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: max_auto_readahead_size: 262144 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: prepopulate_block_cache: 0 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: initial_auto_readahead_size: 8192 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout: num_file_reads_for_auto_readahead: 2 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.write_buffer_size: 33554432 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_write_buffer_number: 2 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression: NoCompression 2024-04-30T09:55:31.746 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression: Disabled 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.prefix_extractor: nullptr 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.num_levels: 7 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-30T09:55:31.747 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression_opts.level: 32767 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression_opts.strategy: 0 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression_opts.enabled: false 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.target_file_size_base: 67108864 2024-04-30T09:55:31.748 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-30T09:55:31.749 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.arena_block_size: 1048576 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.disable_auto_compactions: 0 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.inplace_update_support: 0 2024-04-30T09:55:31.750 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.bloom_locality: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.max_successive_merges: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.paranoid_file_checks: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.force_consistency_checks: 1 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.report_bg_io_stats: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.ttl: 2592000 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.enable_blob_files: false 2024-04-30T09:55:31.751 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.min_blob_size: 0 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.blob_file_size: 268435456 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.blob_file_starting_level: 0 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 57.sst 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 59, last_sequence is 31343, log_number is 55,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 55 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b222fac2-6d29-4672-a965-4e9afe4dd1c1 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714470931585059, "job": 1, "event": "recovery_started", "wal_files": [55]} 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #55 mode 2 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714470931586798, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 60, "file_size": 234183, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 31348, "largest_seqno": 31641, "table_properties": {"data_size": 232191, "index_size": 634, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 3031, "raw_average_key_size": 24, "raw_value_size": 229525, "raw_average_value_size": 1836, "num_data_blocks": 29, "num_entries": 125, "num_filter_entries": 125, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714470931, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b222fac2-6d29-4672-a965-4e9afe4dd1c1", "db_session_id": "CFI5QDN5DW1EBTYN5JAC", "orig_file_number": 60, "seqno_to_time_mapping": "N/A"}} 2024-04-30T09:55:31.752 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714470931586884, "job": 1, "event": "recovery_finished"} 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/version_set.cc:5047] Creating manifest 62 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000055.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55d5f6bb0000 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: DB pointer 0x55d5f6b9c000 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** DB Stats ** 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:55:31.753 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** Compaction Stats [default] ** 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: L0 1/0 228.69 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 218.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: L6 1/0 9.46 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: Sum 2/0 9.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 218.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 218.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** Compaction Stats [default] ** 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 218.7 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:55:31.754 INFO:journalctl@ceph.mon.b.smithi046.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: Cumulative compaction: 0.00 GB write, 54.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: Interval compaction: 0.00 GB write, 54.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: Block cache BinnedLRUCache@0x55d5f6a931f0#2 capacity: 512.00 MB usage: 1.08 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 8e-06 secs_since: 0 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,0.70 KB,0.00013411%) Misc(1,0.00 KB,0%) 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: starting mon.b rank 2 at public addrs [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] at bind addrs [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:55:31.755 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: mon.b@-1(???) e3 preinit fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: mon.b@-1(???).mds e1 new map 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: mon.b@-1(???).mds e1 print_map 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout: e1 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout: legacy client fscid: -1 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout: 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout: No filesystems configured 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: mon.b@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: mon.b@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: mon.b@-1(???).paxosservice(auth 1..25) refresh upgraded, format 0 -> 3 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 bash[65670]: 9bfb997f7d880f28d1c60241527360159d2b018853689f4034a8e63c04290d99 2024-04-30T09:55:31.756 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 systemd[1]: Started Ceph mon.b for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:55:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:31 smithi026 ceph-mon[27123]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:32.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:31 smithi026 ceph-mon[22362]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:32.337 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:31 smithi046 ceph-mon[65689]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-30T09:55:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: mon.b calling monitor election 2024-04-30T09:55:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: mon.c calling monitor election 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: mon.a calling monitor election 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: fsmap 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: osdmap e112: 8 total, 8 up, 8 in 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: mgrmap e41: y(active, since 3m), standbys: x 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:32.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:32 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: mon.b calling monitor election 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: mon.c calling monitor election 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: mon.a calling monitor election 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: fsmap 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: osdmap e112: 8 total, 8 up, 8 in 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: mgrmap e41: y(active, since 3m), standbys: x 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: mon.b calling monitor election 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: mon.c calling monitor election 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: mon.a calling monitor election 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: fsmap 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: osdmap e112: 8 total, 8 up, 8 in 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: mgrmap e41: y(active, since 3m), standbys: x 2024-04-30T09:55:33.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:55:33.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:55:33.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:55:33.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:33.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:33.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:32 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:34.174 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:33 smithi046 ceph-mon[65689]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:55:34.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:33 smithi026 ceph-mon[22362]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:55:34.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:33 smithi026 ceph-mon[27123]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:55:35.061 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:34 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:35.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:34 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:35.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:34 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:35.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:35 smithi046 ceph-mon[65689]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:35 smithi026 ceph-mon[27123]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:35 smithi026 ceph-mon[22362]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:55:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:55:36] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-04-30T09:55:36.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:55:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T09:55:36.594+0000 7f4a84fcd700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-30T09:55:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:36 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:36 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:36 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:37.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:36 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:36 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:36 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:36 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:36 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:37.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:36 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:37.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:37 smithi026 ceph-mon[27123]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:37.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:37 smithi026 ceph-mon[22362]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:37.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:37 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:37.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:37 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:38.196 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:37 smithi046 ceph-mon[65689]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:38.196 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:37 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:38.196 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:37 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:38.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:37 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:38.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:37 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:39 smithi046 ceph-mon[65689]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:40.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:39 smithi026 ceph-mon[22362]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:40.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:39 smithi026 ceph-mon[27123]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:41.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Detected new or changed devices on smithi046 2024-04-30T09:55:41.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all crash 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mds 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nfs 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all node-exporter 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all prometheus 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all alertmanager 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all grafana 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all loki 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:41.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all promtail 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Finalizing container_image settings 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:55:41.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:55:41.989 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: Upgrade: Complete! 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 839 B/s rd, 0 op/s 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.990 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:41.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:41.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:41.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:41.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:55:41.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:41.991 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:41 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Detected new or changed devices on smithi046 2024-04-30T09:55:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-04-30T09:55:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:55:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all crash 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all mds 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T09:55:42.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all nfs 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi046 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all crash 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all mds 2024-04-30T09:55:42.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all nfs 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.201 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:55:42.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all node-exporter 2024-04-30T09:55:42.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all prometheus 2024-04-30T09:55:42.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all alertmanager 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all grafana 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all loki 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all promtail 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Finalizing container_image settings 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:55:42.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all node-exporter 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all prometheus 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all alertmanager 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all grafana 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all loki 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all promtail 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Finalizing container_image settings 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:55:42.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T09:55:42.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: Upgrade: Complete! 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T09:55:42.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: Upgrade: Complete! 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 839 B/s rd, 0 op/s 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 632 B/s rd, 0 op/s 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 839 B/s rd, 0 op/s 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:55:42.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:55:42.211 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:55:42.211 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:42.211 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:55:42.211 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:42.211 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:41 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:55:43.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:43 smithi026 ceph-mon[27123]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-04-30T09:55:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:43 smithi026 ceph-mon[22362]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-04-30T09:55:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:43 smithi046 ceph-mon[65689]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-04-30T09:55:44.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:44 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:44 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:44 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:45.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:45 smithi026 ceph-mon[22362]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:45.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:45 smithi026 ceph-mon[27123]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:45.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:45 smithi046 ceph-mon[65689]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:55:46.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:55:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:55:46] "GET /metrics HTTP/1.1" 200 37927 "" "Prometheus/2.43.0" 2024-04-30T09:55:47.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:47 smithi026 ceph-mon[27123]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-30T09:55:47.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:47 smithi026 ceph-mon[22362]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-30T09:55:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:47 smithi046 ceph-mon[65689]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-30T09:55:49.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:49 smithi026 ceph-mon[22362]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:49.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:49 smithi026 ceph-mon[27123]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:49 smithi046 ceph-mon[65689]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:51 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:51 smithi046 ceph-mon[65689]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:55:52.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:51 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:52.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:51 smithi026 ceph-mon[27123]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:55:52.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:51 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:55:52.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:51 smithi026 ceph-mon[22362]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:55:53.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:53 smithi026 ceph-mon[27123]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:53 smithi026 ceph-mon[22362]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:53 smithi046 ceph-mon[65689]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:54.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:54 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:54 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:54 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:55:55.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:55 smithi026 ceph-mon[27123]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:55.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:55 smithi026 ceph-mon[22362]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:55 smithi046 ceph-mon[65689]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:56.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:55:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:55:56] "GET /metrics HTTP/1.1" 200 37998 "" "Prometheus/2.43.0" 2024-04-30T09:55:57.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T09:55:57.399 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:57 smithi026 ceph-mon[27123]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:57.399 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:57 smithi026 ceph-mon[27123]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:57.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:57 smithi026 ceph-mon[22362]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:57.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:57 smithi026 ceph-mon[22362]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:57 smithi046 ceph-mon[65689]: from='client.25618 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:57 smithi046 ceph-mon[65689]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:55:59.056 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:55:59.056 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 119s ago 24m 15.7M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:55:59.056 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (101s) 22s ago 24m 75.0M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:55:59.056 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 119s ago 23m 61.0M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:55:59.056 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (3m) 22s ago 29m 452M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:55:59.056 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (13m) 119s ago 30m 504M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (30m) 119s ago 30m 79.4M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (27s) 22s ago 29m 17.9M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (29m) 119s ago 29m 60.9M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) 119s ago 25m 8468k - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (2m) 22s ago 25m 16.7M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (28m) 119s ago 28m 53.5M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (28m) 119s ago 28m 54.0M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (27m) 119s ago 27m 53.1M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (27m) 119s ago 27m 52.3M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (27m) 22s ago 27m 58.1M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (26m) 22s ago 26m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (26m) 22s ago 26m 52.1M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (25m) 22s ago 25m 54.3M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:55:59.057 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (2m) 22s ago 25m 67.1M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:55:59.058 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (23m) 119s ago 23m 75.8M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:55:59.058 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (23m) 22s ago 23m 76.2M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:55:59.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:59 smithi026 ceph-mon[27123]: from='client.25624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:59.321 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:55:59 smithi026 ceph-mon[27123]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:59.322 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:59 smithi026 ceph-mon[22362]: from='client.25624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:59.322 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:55:59 smithi026 ceph-mon[22362]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:59 smithi046 ceph-mon[65689]: from='client.25624 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:55:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:55:59 smithi046 ceph-mon[65689]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:55:59.726 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 2'"'"'' 2024-04-30T09:56:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:56:02.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:01 smithi026 ceph-mon[27123]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:02.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:01 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3015985519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:02.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:01 smithi026 ceph-mon[22362]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:02.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:01 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3015985519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:01 smithi046 ceph-mon[65689]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:01 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/3015985519' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:02.594 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade status' 2024-04-30T09:56:03.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:03 smithi026 ceph-mon[27123]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:03.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:03 smithi026 ceph-mon[22362]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:03 smithi046 ceph-mon[65689]: pgmap v131: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:04.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:04 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:04.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:04 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:04.469 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:56:04.470 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-30T09:56:04.470 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-30T09:56:04.470 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-30T09:56:04.470 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:56:04.470 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-30T09:56:04.470 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-30T09:56:04.470 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:56:04.470 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:56:04.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:04 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:05.096 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T09:56:05.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:05 smithi026 ceph-mon[27123]: from='client.15738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:05.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:05 smithi026 ceph-mon[27123]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:05.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:05 smithi026 ceph-mon[22362]: from='client.15738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:05.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:05 smithi026 ceph-mon[22362]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:05 smithi046 ceph-mon[65689]: from='client.15738 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:05 smithi046 ceph-mon[65689]: pgmap v132: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:06.668 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:56:06] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-04-30T09:56:07.194 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:07.194 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:07.194 INFO:teuthology.orchestra.run.smithi026.stdout: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:07.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:06 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:56:07.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:06 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:56:07.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:06 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:56:07.727 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '"'"'{print $2}'"'"')' 2024-04-30T09:56:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:07 smithi046 ceph-mon[65689]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:08.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:07 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/3337353840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:56:08.312 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:07 smithi026 ceph-mon[22362]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:08.312 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:07 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/3337353840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:56:08.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:07 smithi026 ceph-mon[27123]: pgmap v133: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:08.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:07 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/3337353840' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:56:09.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:09 smithi026 ceph-mon[27123]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:09.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:09 smithi026 ceph-mon[22362]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:09 smithi046 ceph-mon[65689]: pgmap v134: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:10.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:10 smithi026 ceph-mon[27123]: from='client.15750 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:10.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:10 smithi026 ceph-mon[22362]: from='client.15750 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:10 smithi046 ceph-mon[65689]: from='client.15750 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:11 smithi046 ceph-mon[65689]: from='client.25654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "mon", "hosts": "smithi026", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:12.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:11 smithi046 ceph-mon[65689]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:12.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:11 smithi026 ceph-mon[27123]: from='client.25654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "mon", "hosts": "smithi026", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:12.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:11 smithi026 ceph-mon[27123]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:12.254 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:11 smithi026 ceph-mon[22362]: from='client.25654 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "mon", "hosts": "smithi026", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:12.254 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:11 smithi026 ceph-mon[22362]: pgmap v135: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:13.128 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:56:13.760 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-30T09:56:14.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[22362]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:56:14.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:14.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[22362]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:14.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[27123]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[27123]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:56:14.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:14 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:14 smithi046 ceph-mon[65689]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:56:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:14 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:14 smithi046 ceph-mon[65689]: pgmap v136: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:14 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:14 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:14 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:56:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:14 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:15.394 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:15 smithi026 ceph-mon[22362]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-04-30T09:56:15.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:15 smithi026 ceph-mon[22362]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:15.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:15 smithi026 ceph-mon[22362]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:56:15.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:15 smithi026 ceph-mon[22362]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:15.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:15 smithi026 ceph-mon[27123]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-04-30T09:56:15.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:15 smithi026 ceph-mon[27123]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:15.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:15 smithi026 ceph-mon[27123]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:56:15.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:15 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:15 smithi046 ceph-mon[65689]: pgmap v137: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1021 B/s rd, 0 op/s 2024-04-30T09:56:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:15 smithi046 ceph-mon[65689]: pgmap v138: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:15 smithi046 ceph-mon[65689]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:56:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:15 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:56:16] "GET /metrics HTTP/1.1" 200 37993 "" "Prometheus/2.43.0" 2024-04-30T09:56:17.103 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:56:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[27123]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-30T09:56:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-30T09:56:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-30T09:56:17.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[22362]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-30T09:56:17.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:17.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:17.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:17.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:17.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-30T09:56:17.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:17 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-30T09:56:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:17 smithi046 ceph-mon[65689]: pgmap v139: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-30T09:56:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:17 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:17 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:17 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:17 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:17 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-30T09:56:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:17 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-30T09:56:17.798 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 2m ago 24m 15.7M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (2m) 41s ago 24m 75.0M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 2m ago 23m 61.0M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (3m) 41s ago 29m 452M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (13m) 2m ago 31m 504M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (31m) 2m ago 31m 79.4M 2048M 17.2.0 e1d6a67b021e b6420404af4c 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (46s) 41s ago 29m 17.9M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (29m) 2m ago 29m 60.9M 2048M 17.2.0 e1d6a67b021e 4717371f2f81 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) 2m ago 25m 8468k - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (3m) 41s ago 25m 16.7M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (29m) 2m ago 29m 53.5M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (28m) 2m ago 28m 54.0M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:56:17.799 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (28m) 2m ago 28m 53.1M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:56:17.800 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (27m) 2m ago 27m 52.3M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:56:17.800 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (27m) 41s ago 27m 58.1M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:56:17.800 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (27m) 41s ago 27m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:56:17.800 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (26m) 41s ago 26m 52.1M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:56:17.800 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (26m) 41s ago 26m 54.3M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:56:17.800 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (2m) 41s ago 25m 67.1M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:56:17.800 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (23m) 2m ago 23m 75.8M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:56:17.800 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (23m) 41s ago 23m 76.2M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:56:18.222 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:56:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:56:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-30T09:56:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 1 2024-04-30T09:56:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": {}, 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 12, 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:56:18.223 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:56:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[27123]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:56:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[27123]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:56:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[27123]: Upgrade: Setting container_image for all mgr 2024-04-30T09:56:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[27123]: Upgrade: It appears safe to stop mon.a 2024-04-30T09:56:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[27123]: from='client.25660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[22362]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:56:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[22362]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:56:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[22362]: Upgrade: Setting container_image for all mgr 2024-04-30T09:56:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[22362]: Upgrade: It appears safe to stop mon.a 2024-04-30T09:56:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:18 smithi026 ceph-mon[22362]: from='client.25660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:18 smithi046 ceph-mon[65689]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:56:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:18 smithi046 ceph-mon[65689]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:56:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:18 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mgr 2024-04-30T09:56:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:18 smithi046 ceph-mon[65689]: Upgrade: It appears safe to stop mon.a 2024-04-30T09:56:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:18 smithi046 ceph-mon[65689]: from='client.25660 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:18.604 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:56:18.604 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:56:18.604 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:56:18.604 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi026", 2024-04-30T09:56:18.604 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:56:18.604 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "0/2 daemons upgraded", 2024-04-30T09:56:18.604 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mon daemons", 2024-04-30T09:56:18.604 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:56:18.604 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:56:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: from='client.15768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: from='client.15774 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: Upgrade: Updating mon.a 2024-04-30T09:56:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:56:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:56:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:19.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: Deploying daemon mon.a on smithi026 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: from='client.? 172.21.15.26:0/136734369' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[27123]: from='client.25684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: from='client.15768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: from='client.15774 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: Upgrade: Updating mon.a 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: Deploying daemon mon.a on smithi026 2024-04-30T09:56:19.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: from='client.? 172.21.15.26:0/136734369' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:19.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:19 smithi026 ceph-mon[22362]: from='client.25684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: pgmap v140: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: from='client.15768 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: from='client.15774 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: Upgrade: Updating mon.a 2024-04-30T09:56:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' 2024-04-30T09:56:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:56:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:56:19.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:19.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: Deploying daemon mon.a on smithi026 2024-04-30T09:56:19.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/136734369' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:19.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:19 smithi046 ceph-mon[65689]: from='client.25684 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:20.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-mon[27123]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:20.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:20 smithi026 systemd[1]: Stopping Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:56:20.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-mon[22362]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:20 smithi046 ceph-mon[65689]: pgmap v141: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:20.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a[22358]: 2024-04-30T09:56:20.323+0000 7f396ec26700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T09:56:20.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a[22358]: 2024-04-30T09:56:20.323+0000 7f396ec26700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-30T09:56:20.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:20 smithi026 podman[80791]: 2024-04-30 09:56:20.548329172 +0000 UTC m=+0.353636320 container died b6420404af4cdb8d1fe0e2b9186e5a1d95b7549bfc023e1fb5ce49531f43c725 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, vcs-type=git, com.redhat.component=centos-stream-container, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.buildah.version=1.19.8, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, io.openshift.expose-services=, release=754, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, maintainer=Guillaume Abrioux ) 2024-04-30T09:56:20.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:56:20] ENGINE Bus STOPPING 2024-04-30T09:56:20.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:56:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T09:56:20.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:56:20] ENGINE Bus STOPPED 2024-04-30T09:56:20.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:56:20] ENGINE Bus STARTING 2024-04-30T09:56:20.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:56:20] ENGINE Serving on http://:::9283 2024-04-30T09:56:20.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:20 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:56:20] ENGINE Bus STARTED 2024-04-30T09:56:21.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:20 smithi026 podman[80791]: 2024-04-30 09:56:20.915115133 +0000 UTC m=+0.720422323 container cleanup b6420404af4cdb8d1fe0e2b9186e5a1d95b7549bfc023e1fb5ce49531f43c725 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, com.redhat.component=centos-stream-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, GIT_CLEAN=True, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, release=754, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, io.openshift.expose-services=, ceph=True, vendor=Red Hat, Inc., architecture=x86_64, maintainer=Guillaume Abrioux ) 2024-04-30T09:56:21.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:20 smithi026 bash[80791]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a 2024-04-30T09:56:21.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 podman[80817]: 2024-04-30 09:56:21.183097686 +0000 UTC m=+0.638356453 container remove b6420404af4cdb8d1fe0e2b9186e5a1d95b7549bfc023e1fb5ce49531f43c725 (image=quay.io/ceph/ceph:v17.2.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, build-date=2022-05-03T08:36:31.336870, release=754, architecture=x86_64, vcs-type=git, GIT_CLEAN=True, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., distribution-scope=public, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, CEPH_POINT_RELEASE=-17.2.0, version=8, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.buildah.version=1.19.8, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container) 2024-04-30T09:56:21.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-30T09:56:21.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 systemd[1]: Stopped Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:56:21.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.a.service: Consumed 28.789s CPU time. 2024-04-30T09:56:21.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 systemd[1]: Starting Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:56:21.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 podman[80950]: 2024-04-30 09:56:21.759163866 +0000 UTC m=+0.109956226 container create 649ef76a25d18a8abaa771a4b8d435bf3bbac0c5fa1da504be5a45589e6f4f56 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T09:56:21.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 podman[80950]: 2024-04-30 09:56:21.667955405 +0000 UTC m=+0.018747790 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 podman[80950]: 2024-04-30 09:56:21.970451744 +0000 UTC m=+0.321244118 container init 649ef76a25d18a8abaa771a4b8d435bf3bbac0c5fa1da504be5a45589e6f4f56 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 podman[80950]: 2024-04-30 09:56:21.973771278 +0000 UTC m=+0.324563639 container start 649ef76a25d18a8abaa771a4b8d435bf3bbac0c5fa1da504be5a45589e6f4f56 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True) 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 ceph-mon[80969]: set uid:gid to 167:167 (ceph:ceph) 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 ceph-mon[80969]: ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev), process ceph-mon, pid 2 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:21 smithi026 ceph-mon[80969]: pidfile_write: ignore empty --pid-file 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: load: jerasure load: lrc 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: RocksDB version: 7.9.2 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Git sha 0 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Compile date 2024-04-30 00:29:27 2024-04-30T09:56:22.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: DB SUMMARY 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: DB Session ID: U6LVUONSUV71SCNP6ILP 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: CURRENT file: CURRENT 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: IDENTITY file: IDENTITY 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: MANIFEST file: MANIFEST-000015 size: 3125 Bytes 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000063.sst 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000061.log size: 1181651 ; 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.error_if_exists: 0 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.create_if_missing: 0 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.paranoid_checks: 1 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-30T09:56:22.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.env: 0x559468289cc0 2024-04-30T09:56:22.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.fs: PosixFileSystem 2024-04-30T09:56:22.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.info_log: 0x559469fbbda0 2024-04-30T09:56:22.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_file_opening_threads: 16 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.statistics: (nil) 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.use_fsync: 0 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_log_file_size: 0 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.keep_log_file_num: 1000 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.recycle_log_file_num: 0 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.allow_fallocate: 1 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.allow_mmap_reads: 0 2024-04-30T09:56:22.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.allow_mmap_writes: 0 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.use_direct_reads: 0 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.create_missing_column_families: 0 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.db_log_dir: 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.wal_dir: 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.advise_random_on_open: 1 2024-04-30T09:56:22.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.db_write_buffer_size: 0 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.write_buffer_manager: 0x559469f8d0e0 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.rate_limiter: (nil) 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.wal_recovery_mode: 2 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.enable_thread_tracking: 0 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.enable_pipelined_write: 0 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.unordered_write: 0 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-30T09:56:22.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.row_cache: None 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.wal_filter: None 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.allow_ingest_behind: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.two_write_queues: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.manual_wal_flush: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.wal_compression: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.atomic_flush: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.log_readahead_size: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.best_efforts_recovery: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.allow_data_in_errors: 0 2024-04-30T09:56:22.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.db_host_id: __hostname__ 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_background_jobs: 2 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_background_compactions: -1 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_subcompactions: 1 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_total_wal_size: 0 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_open_files: -1 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bytes_per_sync: 0 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-30T09:56:22.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_readahead_size: 0 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_background_flushes: -1 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Compression algorithms supported: 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: kZSTD supported: 0 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: kXpressCompression supported: 0 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: kLZ4HCCompression supported: 1 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: kZlibCompression supported: 1 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: kSnappyCompression supported: 1 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: kLZ4Compression supported: 1 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: kBZip2Compression supported: 0 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-30T09:56:22.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.merge_operator: 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_filter: None 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_filter_factory: None 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.sst_partitioner_factory: None 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559469fbbf60) 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2024-04-30T09:56:22.206 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 4 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x559469fbf1f0 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2024-04-30T09:56:22.207 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: bloomfilter 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 5 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_auto_readahead_size: 262144 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: prepopulate_block_cache: 0 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: initial_auto_readahead_size: 8192 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.write_buffer_size: 33554432 2024-04-30T09:56:22.208 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_write_buffer_number: 2 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression: NoCompression 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression: Disabled 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.prefix_extractor: nullptr 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.num_levels: 7 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-30T09:56:22.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression_opts.level: 32767 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression_opts.strategy: 0 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression_opts.enabled: false 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-30T09:56:22.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.target_file_size_base: 67108864 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-30T09:56:22.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.arena_block_size: 1048576 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.disable_auto_compactions: 0 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-30T09:56:22.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.inplace_update_support: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.bloom_locality: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.max_successive_merges: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.paranoid_file_checks: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.force_consistency_checks: 1 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.report_bg_io_stats: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.ttl: 2592000 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.enable_blob_files: false 2024-04-30T09:56:22.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.min_blob_size: 0 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.blob_file_size: 268435456 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.blob_file_starting_level: 0 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 65, last_sequence is 28991, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 94a0d9a8-7240-4157-afeb-38bd9e2fe26a 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714470982014129, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-04-30T09:56:22.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714470982019092, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 1109048, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 28992, "largest_seqno": 30230, "table_properties": {"data_size": 1103938, "index_size": 2662, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1413, "raw_key_size": 14258, "raw_average_key_size": 25, "raw_value_size": 1092527, "raw_average_value_size": 1950, "num_data_blocks": 120, "num_entries": 560, "num_filter_entries": 560, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714470982, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "94a0d9a8-7240-4157-afeb-38bd9e2fe26a", "db_session_id": "U6LVUONSUV71SCNP6ILP", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714470982019160, "job": 1, "event": "recovery_finished"} 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55946a0dc000 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: DB pointer 0x55946a0c8000 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-30T09:56:22.215 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 1/0 1.06 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 388.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: L6 1/0 9.46 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 10.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 388.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 388.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:56:22.216 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 388.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 58.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 58.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache BinnedLRUCache@0x559469fbf1f0#2 capacity: 512.00 MB usage: 4.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2024-04-30T09:56:22.217 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: mon.a@-1(???) e3 preinit fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: mon.a@-1(???).mds e1 new map 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: mon.a@-1(???).mds e1 print_map 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout: e1 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: -1 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout: No filesystems configured 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: mon.a@-1(???).osd e112 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-30T09:56:22.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: mon.a@-1(???).osd e112 crush map has features 432629239337189376, adjusting msgr requires 2024-04-30T09:56:22.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 ceph-mon[80969]: mon.a@-1(???).paxosservice(auth 1..26) refresh upgraded, format 0 -> 3 2024-04-30T09:56:22.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 bash[80950]: 649ef76a25d18a8abaa771a4b8d435bf3bbac0c5fa1da504be5a45589e6f4f56 2024-04-30T09:56:22.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:22 smithi026 systemd[1]: Started Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:56:23.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ignoring --setuser ceph since I am not root 2024-04-30T09:56:23.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ignoring --setgroup ceph since I am not root 2024-04-30T09:56:23.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mgr[60498]: -- 172.21.15.26:0/3070257423 <== mon.1 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5596651b45a0 con 0x559665188c00 2024-04-30T09:56:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:56:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:56:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:56:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:56:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-30T09:56:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: mon.a calling monitor election 2024-04-30T09:56:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: monmap epoch 3 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: last_changed 2024-04-30T09:26:37.291976+0000 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: created 2024-04-30T09:24:58.133666+0000 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: min_mon_release 17 (quincy) 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: election_strategy: 1 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: 2: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.b 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: fsmap 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: osdmap e112: 8 total, 8 up, 8 in 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: mgrmap e41: y(active, since 4m), standbys: x 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:23.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: from='mgr.25240 ' entity='' 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[80969]: mgrmap e42: y(active, since 4m), standbys: x 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: mon.a calling monitor election 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:56:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: monmap epoch 3 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: last_changed 2024-04-30T09:26:37.291976+0000 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: created 2024-04-30T09:24:58.133666+0000 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: min_mon_release 17 (quincy) 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: election_strategy: 1 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: 2: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.b 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: fsmap 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: osdmap e112: 8 total, 8 up, 8 in 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: mgrmap e41: y(active, since 4m), standbys: x 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:23.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:23.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:23.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: from='mgr.25240 ' entity='' 2024-04-30T09:56:23.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:23 smithi026 ceph-mon[27123]: mgrmap e42: y(active, since 4m), standbys: x 2024-04-30T09:56:23.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: ignoring --setuser ceph since I am not root 2024-04-30T09:56:23.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: ignoring --setgroup ceph since I am not root 2024-04-30T09:56:23.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mgr[58621]: -- 172.21.15.46:0/2579684797 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b57c0165a0 con 0x55b57bfea800 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: from='mgr.25240 172.21.15.26:0/2276822303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: pgmap v142: 161 pgs: 161 active+clean; 585 KiB data, 98 MiB used, 715 GiB / 715 GiB avail; 638 B/s rd, 0 op/s 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: mon.a calling monitor election 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: monmap epoch 3 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: last_changed 2024-04-30T09:26:37.291976+0000 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: created 2024-04-30T09:24:58.133666+0000 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: min_mon_release 17 (quincy) 2024-04-30T09:56:23.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: election_strategy: 1 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: 2: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.b 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: fsmap 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: osdmap e112: 8 total, 8 up, 8 in 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: mgrmap e41: y(active, since 4m), standbys: x 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: from='mgr.25240 ' entity='' 2024-04-30T09:56:23.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:23 smithi046 ceph-mon[65689]: mgrmap e42: y(active, since 4m), standbys: x 2024-04-30T09:56:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:25 smithi046 ceph-mon[65689]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-30T09:56:25.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:25 smithi026 ceph-mon[27123]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-30T09:56:25.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:25 smithi026 ceph-mon[80969]: mgrmap e43: y(active, since 4m), standbys: x 2024-04-30T09:56:30.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: [30/Apr/2024:09:56:30] ENGINE Bus STARTING 2024-04-30T09:56:30.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: CherryPy Checker: 2024-04-30T09:56:30.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: The Application mounted at '' has an empty config. 2024-04-30T09:56:30.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: 2024-04-30T09:56:30.984 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: [30/Apr/2024:09:56:30] ENGINE Serving on http://:::9283 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x[58617]: [30/Apr/2024:09:56:30] ENGINE Bus STARTED 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: Active manager daemon y restarted 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: Activating manager daemon y 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: osdmap e113: 8 total, 8 up, 8 in 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: Standby manager daemon x restarted 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: Standby manager daemon x started 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: mgrmap e44: y(active, starting, since 0.00786738s), standbys: x 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:56:30.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:56:30.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:56:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:56:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:56:30.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:30 smithi046 ceph-mon[65689]: Manager daemon y is now available 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:56:30] ENGINE Bus STARTING 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: CherryPy Checker: 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: The Application mounted at '' has an empty config. 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:56:30] ENGINE Serving on http://:::9283 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:09:56:30] ENGINE Bus STARTED 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: Active manager daemon y restarted 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: Activating manager daemon y 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: osdmap e113: 8 total, 8 up, 8 in 2024-04-30T09:56:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: Standby manager daemon x restarted 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: Standby manager daemon x started 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: mgrmap e44: y(active, starting, since 0.00786738s), standbys: x 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:56:31.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[27123]: Manager daemon y is now available 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: Active manager daemon y restarted 2024-04-30T09:56:31.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: Activating manager daemon y 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: osdmap e113: 8 total, 8 up, 8 in 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: Standby manager daemon x restarted 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: Standby manager daemon x started 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: mgrmap e44: y(active, starting, since 0.00786738s), standbys: x 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.? 172.21.15.46:0/1476616716' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:56:31.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-30T09:56:31.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:30 smithi026 ceph-mon[80969]: Manager daemon y is now available 2024-04-30T09:56:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:56:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:56:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:56:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:31 smithi046 ceph-mon[65689]: mgrmap e45: y(active, since 1.01093s), standbys: x 2024-04-30T09:56:32.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:32.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:56:32.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:56:32.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:56:32.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[27123]: mgrmap e45: y(active, since 1.01093s), standbys: x 2024-04-30T09:56:32.060 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T09:56:31.695+0000 7f3e7839e700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-30T09:56:32.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:32.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:56:32.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-30T09:56:32.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-30T09:56:32.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:31 smithi026 ceph-mon[80969]: mgrmap e45: y(active, since 1.01093s), standbys: x 2024-04-30T09:56:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:32 smithi046 ceph-mon[65689]: [30/Apr/2024:09:56:31] ENGINE Bus STARTING 2024-04-30T09:56:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:32 smithi046 ceph-mon[65689]: [30/Apr/2024:09:56:31] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:56:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:32 smithi046 ceph-mon[65689]: [30/Apr/2024:09:56:32] ENGINE Serving on http://172.21.15.26:8765 2024-04-30T09:56:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:32 smithi046 ceph-mon[65689]: [30/Apr/2024:09:56:32] ENGINE Bus STARTED 2024-04-30T09:56:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:32 smithi046 ceph-mon[65689]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state']) 2024-04-30T09:56:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:32 smithi046 ceph-mon[65689]: Cluster is now healthy 2024-04-30T09:56:33.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[27123]: [30/Apr/2024:09:56:31] ENGINE Bus STARTING 2024-04-30T09:56:33.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[27123]: [30/Apr/2024:09:56:31] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[27123]: [30/Apr/2024:09:56:32] ENGINE Serving on http://172.21.15.26:8765 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[27123]: [30/Apr/2024:09:56:32] ENGINE Bus STARTED 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[27123]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state']) 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[27123]: Cluster is now healthy 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[80969]: [30/Apr/2024:09:56:31] ENGINE Bus STARTING 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[80969]: [30/Apr/2024:09:56:31] ENGINE Serving on https://172.21.15.26:7150 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[80969]: [30/Apr/2024:09:56:32] ENGINE Serving on http://172.21.15.26:8765 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[80969]: [30/Apr/2024:09:56:32] ENGINE Bus STARTED 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[80969]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state']) 2024-04-30T09:56:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:32 smithi026 ceph-mon[80969]: Cluster is now healthy 2024-04-30T09:56:33.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:33 smithi026 ceph-mon[27123]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:56:33.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:33 smithi026 ceph-mon[27123]: mgrmap e46: y(active, since 2s), standbys: x 2024-04-30T09:56:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:33 smithi026 ceph-mon[80969]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:56:33.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:33 smithi026 ceph-mon[80969]: mgrmap e46: y(active, since 2s), standbys: x 2024-04-30T09:56:34.098 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:33 smithi046 ceph-mon[65689]: pgmap v4: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:56:34.098 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:33 smithi046 ceph-mon[65689]: mgrmap e46: y(active, since 2s), standbys: x 2024-04-30T09:56:36.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[27123]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:56:36.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[27123]: mgrmap e47: y(active, since 4s), standbys: x 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[80969]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[80969]: mgrmap e47: y(active, since 4s), standbys: x 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:35 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:35 smithi046 ceph-mon[65689]: pgmap v5: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:56:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:35 smithi046 ceph-mon[65689]: mgrmap e47: y(active, since 4s), standbys: x 2024-04-30T09:56:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:35 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:35 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:35 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:35 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:56:36] "GET /metrics HTTP/1.1" 200 35202 "" "Prometheus/2.43.0" 2024-04-30T09:56:37.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:37 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:37 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:37 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:37 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:37 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:37 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:37 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:37 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:37 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:37 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:37 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:37.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:37 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:38.539 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:38 smithi026 ceph-mon[27123]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-30T09:56:38.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:38 smithi026 ceph-mon[80969]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-30T09:56:38.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:38 smithi046 ceph-mon[65689]: pgmap v6: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-30T09:56:40.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[27123]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:56:40.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[80969]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:40.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:56:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:40 smithi046 ceph-mon[65689]: pgmap v7: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 17 op/s 2024-04-30T09:56:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:56:41.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[27123]: Detected new or changed devices on smithi026 2024-04-30T09:56:41.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:56:41.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:56:41.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[80969]: Detected new or changed devices on smithi026 2024-04-30T09:56:41.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[80969]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:56:41.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[80969]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:56:41.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:41 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:41 smithi046 ceph-mon[65689]: Detected new or changed devices on smithi026 2024-04-30T09:56:41.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:41 smithi046 ceph-mon[65689]: Updating smithi026:/etc/ceph/ceph.conf 2024-04-30T09:56:41.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:41 smithi046 ceph-mon[65689]: Updating smithi046:/etc/ceph/ceph.conf 2024-04-30T09:56:41.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:41 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:41 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:41 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:41 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:41.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:41 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:42.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T09:56:42.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-30T09:56:42.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:42 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-30T09:56:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:56:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.conf 2024-04-30T09:56:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: pgmap v8: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-04-30T09:56:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: Updating smithi046:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:56:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2024-04-30T09:56:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: Updating smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:56:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: Updating smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/config/ceph.client.admin.keyring 2024-04-30T09:56:42.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: pgmap v9: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 22 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-30T09:56:42.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T09:56:42.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:42.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:42.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-30T09:56:42.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:42 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-30T09:56:43.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:43 smithi046 ceph-mon[65689]: Upgrade: It appears safe to stop mon.c 2024-04-30T09:56:43.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:43 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:43.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:43.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:56:43.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:56:43.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:43.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[27123]: Upgrade: It appears safe to stop mon.c 2024-04-30T09:56:43.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[27123]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:43.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:43.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:56:43.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:56:43.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[27123]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:43.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[80969]: Upgrade: It appears safe to stop mon.c 2024-04-30T09:56:43.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:43.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:43.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:56:43.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:56:43.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:44.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:44 smithi026 ceph-mon[80969]: Upgrade: Updating mon.c 2024-04-30T09:56:44.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:44 smithi026 ceph-mon[80969]: Deploying daemon mon.c on smithi026 2024-04-30T09:56:44.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:44 smithi026 ceph-mon[80969]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:56:44.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:44 smithi026 ceph-mon[27123]: Upgrade: Updating mon.c 2024-04-30T09:56:44.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:44 smithi026 ceph-mon[27123]: Deploying daemon mon.c on smithi026 2024-04-30T09:56:44.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:44 smithi026 ceph-mon[27123]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:56:44.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:44 smithi046 ceph-mon[65689]: Upgrade: Updating mon.c 2024-04-30T09:56:44.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:44 smithi046 ceph-mon[65689]: Deploying daemon mon.c on smithi026 2024-04-30T09:56:44.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:44 smithi046 ceph-mon[65689]: pgmap v10: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 11 op/s 2024-04-30T09:56:45.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:44 smithi026 systemd[1]: Stopping Ceph mon.c for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:56:45.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:45 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c[27119]: 2024-04-30T09:56:45.117+0000 7fab18fc1700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T09:56:45.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:45 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c[27119]: 2024-04-30T09:56:45.117+0000 7fab18fc1700 -1 mon.c@1(peon) e3 *** Got Signal Terminated *** 2024-04-30T09:56:45.678 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:45 smithi026 podman[86143]: 2024-04-30 09:56:45.394197886 +0000 UTC m=+0.368202009 container died 4717371f2f81481949518f2cd43126f60ad77bd9297f7bf8818627e34f00a298 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, architecture=x86_64, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, ceph=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base centos centos-stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., release=754, com.redhat.component=centos-stream-container, maintainer=Guillaume Abrioux , GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream) 2024-04-30T09:56:45.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:45 smithi026 podman[86143]: 2024-04-30 09:56:45.7598585 +0000 UTC m=+0.733862637 container cleanup 4717371f2f81481949518f2cd43126f60ad77bd9297f7bf8818627e34f00a298 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, build-date=2022-05-03T08:36:31.336870, name=centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, vcs-type=git, distribution-scope=public, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD) 2024-04-30T09:56:45.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:45 smithi026 bash[86143]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c 2024-04-30T09:56:46.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 podman[86157]: 2024-04-30 09:56:46.044599772 +0000 UTC m=+0.651164431 container remove 4717371f2f81481949518f2cd43126f60ad77bd9297f7bf8818627e34f00a298 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c, vendor=Red Hat, Inc., architecture=x86_64, build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_BRANCH=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.openshift.expose-services=, release=754, CEPH_POINT_RELEASE=-17.2.0, version=8, RELEASE=HEAD, distribution-scope=public, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-30T09:56:46.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-30T09:56:46.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 systemd[1]: Stopped Ceph mon.c for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:56:46.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.c.service: Consumed 17.962s CPU time. 2024-04-30T09:56:46.654 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:56:46] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.43.0" 2024-04-30T09:56:46.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 systemd[1]: Starting Ceph mon.c for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:56:46.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 podman[86291]: 2024-04-30 09:56:46.517201179 +0000 UTC m=+0.019263418 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:56:46.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 podman[86291]: 2024-04-30 09:56:46.654415504 +0000 UTC m=+0.156477740 container create 44d33511609ff073ddb66ef9874b7cdea20a7cc2065b5a0819f2683f20375938 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-04-30T09:56:46.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 podman[86291]: 2024-04-30 09:56:46.840701111 +0000 UTC m=+0.342763363 container init 44d33511609ff073ddb66ef9874b7cdea20a7cc2065b5a0819f2683f20375938 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 podman[86291]: 2024-04-30 09:56:46.844188172 +0000 UTC m=+0.346250418 container start 44d33511609ff073ddb66ef9874b7cdea20a7cc2065b5a0819f2683f20375938 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: set uid:gid to 167:167 (ceph:ceph) 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev), process ceph-mon, pid 2 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: pidfile_write: ignore empty --pid-file 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: load: jerasure load: lrc 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: RocksDB version: 7.9.2 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Git sha 0 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Compile date 2024-04-30 00:29:27 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: DB SUMMARY 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: DB Session ID: FMTD6DYFNCETN3XCJQP6 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: CURRENT file: CURRENT 2024-04-30T09:56:46.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: IDENTITY file: IDENTITY 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: MANIFEST file: MANIFEST-000009 size: 3431 Bytes 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000063.sst 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000061.log size: 2451570 ; 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.error_if_exists: 0 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.create_if_missing: 0 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.paranoid_checks: 1 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.flush_verify_memtable_count: 1 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.env: 0x55817bfb7cc0 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.fs: PosixFileSystem 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.info_log: 0x55817e80dda0 2024-04-30T09:56:46.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_file_opening_threads: 16 2024-04-30T09:56:46.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.statistics: (nil) 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.use_fsync: 0 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_log_file_size: 0 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.keep_log_file_num: 1000 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.recycle_log_file_num: 0 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.allow_fallocate: 1 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.allow_mmap_reads: 0 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.allow_mmap_writes: 0 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.use_direct_reads: 0 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.create_missing_column_families: 0 2024-04-30T09:56:46.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.db_log_dir: 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.wal_dir: 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.advise_random_on_open: 1 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.db_write_buffer_size: 0 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.write_buffer_manager: 0x55817e7df0e0 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.rate_limiter: (nil) 2024-04-30T09:56:46.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.wal_recovery_mode: 2 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.enable_thread_tracking: 0 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.enable_pipelined_write: 0 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.unordered_write: 0 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.row_cache: None 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.wal_filter: None 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.allow_ingest_behind: 0 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.two_write_queues: 0 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.manual_wal_flush: 0 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.wal_compression: 0 2024-04-30T09:56:46.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.atomic_flush: 0 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.log_readahead_size: 0 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.best_efforts_recovery: 0 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.allow_data_in_errors: 0 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.db_host_id: __hostname__ 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.enforce_single_del_contracts: true 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_background_jobs: 2 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_background_compactions: -1 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_subcompactions: 1 2024-04-30T09:56:46.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_total_wal_size: 0 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_open_files: -1 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bytes_per_sync: 0 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_readahead_size: 0 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_background_flushes: -1 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Compression algorithms supported: 2024-04-30T09:56:46.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: kZSTD supported: 0 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: kXpressCompression supported: 0 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: kLZ4HCCompression supported: 1 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: kZlibCompression supported: 1 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: kSnappyCompression supported: 1 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: kLZ4Compression supported: 1 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: kBZip2Compression supported: 0 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: DMutex implementation: pthread_mutex_t 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.merge_operator: 2024-04-30T09:56:46.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_filter: None 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_filter_factory: None 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.sst_partitioner_factory: None 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55817e80df60) 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks: 1 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_top_level_index_and_filter: 1 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_type: 0 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_index_type: 0 2024-04-30T09:56:46.956 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_shortening: 1 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: checksum: 4 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: no_block_cache: 0 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache: 0x55817e8111f0 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_name: BinnedLRUCache 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_options: 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: capacity : 536870912 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_shard_bits : 4 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: strict_capacity_limit : 0 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: high_pri_pool_ratio: 0.000 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_compressed: (nil) 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: persistent_cache: (nil) 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size: 4096 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size_deviation: 10 2024-04-30T09:56:46.957 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_restart_interval: 16 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_block_restart_interval: 1 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_block_size: 4096 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: partition_filters: 0 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_delta_encoding: 1 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: filter_policy: bloomfilter 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: whole_key_filtering: 1 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: verify_compression: 0 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: read_amp_bytes_per_bit: 0 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: format_version: 5 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_index_compression: 1 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_align: 0 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_auto_readahead_size: 262144 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: prepopulate_block_cache: 0 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: initial_auto_readahead_size: 8192 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2024-04-30T09:56:46.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.write_buffer_size: 33554432 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_write_buffer_number: 2 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression: NoCompression 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression: Disabled 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.prefix_extractor: nullptr 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.num_levels: 7 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:56:46.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression_opts.level: 32767 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression_opts.strategy: 0 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-30T09:56:46.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression_opts.enabled: false 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.target_file_size_base: 67108864 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-30T09:56:46.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.arena_block_size: 1048576 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.disable_auto_compactions: 0 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-30T09:56:46.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.inplace_update_support: 0 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.bloom_locality: 0 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.max_successive_merges: 0 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-30T09:56:46.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.paranoid_file_checks: 0 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.force_consistency_checks: 1 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.report_bg_io_stats: 0 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.ttl: 2592000 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.enable_blob_files: false 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.min_blob_size: 0 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.blob_file_size: 268435456 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-30T09:56:46.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.blob_file_starting_level: 0 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 63.sst 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 65, last_sequence is 33539, log_number is 61,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 61 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e2932d09-5aad-42c7-a0f6-900e0284b06e 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714471006874294, "job": 1, "event": "recovery_started", "wal_files": [61]} 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #61 mode 2 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714471006881135, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 66, "file_size": 1547398, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 33545, "largest_seqno": 33963, "table_properties": {"data_size": 1544452, "index_size": 1394, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 517, "raw_key_size": 4790, "raw_average_key_size": 24, "raw_value_size": 1539925, "raw_average_value_size": 7738, "num_data_blocks": 63, "num_entries": 199, "num_filter_entries": 199, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1714471006, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e2932d09-5aad-42c7-a0f6-900e0284b06e", "db_session_id": "FMTD6DYFNCETN3XCJQP6", "orig_file_number": 66, "seqno_to_time_mapping": "N/A"}} 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: EVENT_LOG_v1 {"time_micros": 1714471006881233, "job": 1, "event": "recovery_finished"} 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/version_set.cc:5047] Creating manifest 68 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-30T09:56:46.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000061.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55817e92e000 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: DB pointer 0x55817e91a000 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** DB Stats ** 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-30T09:56:46.966 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 1.48 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 474.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: L6 1/0 10.80 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 2/0 12.27 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 474.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 474.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 474.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-30T09:56:46.967 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 160.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 160.42 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache BinnedLRUCache@0x55817e8111f0#2 capacity: 512.00 MB usage: 2.05 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.59 KB,0.000113249%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: starting mon.c rank 1 at public addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] at bind addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: mon.c@-1(???) e3 preinit fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:46.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: mon.c@-1(???).mds e1 new map 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: mon.c@-1(???).mds e1 print_map 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout: e1 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout: legacy client fscid: -1 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout: No filesystems configured 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: mon.c@-1(???).osd e113 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: mon.c@-1(???).osd e113 crush map has features 432629239337189376, adjusting msgr requires 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 ceph-mon[86310]: mon.c@-1(???).paxosservice(auth 1..27) refresh upgraded, format 0 -> 3 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 bash[86291]: 44d33511609ff073ddb66ef9874b7cdea20a7cc2065b5a0819f2683f20375938 2024-04-30T09:56:46.969 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:46 smithi026 systemd[1]: Started Ceph mon.c for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:56:48.981 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: 2: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.b 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: fsmap 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: osdmap e113: 8 total, 8 up, 8 in 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: mgrmap e47: y(active, since 17s), standbys: x 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: mon.b calling monitor election 2024-04-30T09:56:49.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: mon.c calling monitor election 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: mon.a calling monitor election 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: monmap epoch 4 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: last_changed 2024-04-30T09:56:48.449092+0000 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: created 2024-04-30T09:24:58.133666+0000 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: min_mon_release 19 (squid) 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: election_strategy: 1 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-30T09:56:49.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: 2: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.b 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: fsmap 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: osdmap e113: 8 total, 8 up, 8 in 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: mgrmap e47: y(active, since 17s), standbys: x 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-30T09:56:49.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: 2: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.b 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: fsmap 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: osdmap e113: 8 total, 8 up, 8 in 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: mgrmap e47: y(active, since 17s), standbys: x 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: mon.b calling monitor election 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: mon.c calling monitor election 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: mon.a calling monitor election 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: monmap epoch 4 2024-04-30T09:56:49.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: last_changed 2024-04-30T09:56:48.449092+0000 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: created 2024-04-30T09:24:58.133666+0000 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: min_mon_release 19 (squid) 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: election_strategy: 1 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: 2: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.b 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: fsmap 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: osdmap e113: 8 total, 8 up, 8 in 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: mgrmap e47: y(active, since 17s), standbys: x 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:49.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:49.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:49.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:49 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:49.705 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:56:49.705 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) 14s ago 25m 26.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:56:49.705 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (2m) 14s ago 25m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:56:49.705 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 14s ago 24m 61.3M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:56:49.705 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (4m) 14s ago 29m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:56:49.705 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (14m) 14s ago 31m 474M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:56:49.705 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (28s) 14s ago 31m 32.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T09:56:49.705 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (78s) 14s ago 30m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 starting - - - 2048M 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) 14s ago 26m 15.9M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (3m) 14s ago 25m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (29m) 14s ago 29m 53.8M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (29m) 14s ago 29m 54.1M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (28m) 14s ago 28m 53.7M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (28m) 14s ago 28m 52.9M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (28m) 14s ago 28m 58.3M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (27m) 14s ago 27m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (27m) 14s ago 27m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (26m) 14s ago 26m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (3m) 14s ago 26m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (24m) 14s ago 24m 76.4M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:56:49.706 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (24m) 14s ago 24m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:56:49.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-30T09:56:49.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: 2: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.b 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: fsmap 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: osdmap e113: 8 total, 8 up, 8 in 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: mgrmap e47: y(active, since 17s), standbys: x 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: mon.b calling monitor election 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: mon.c calling monitor election 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: mon.a calling monitor election 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: mon.a is new leader, mons a,c,b in quorum (ranks 0,1,2) 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-30T09:56:49.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: monmap epoch 4 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: fsid 703f4aea-06d3-11ef-bc93-c7b262605968 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: last_changed 2024-04-30T09:56:48.449092+0000 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: created 2024-04-30T09:24:58.133666+0000 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: min_mon_release 19 (squid) 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: election_strategy: 1 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: 0: [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon.a 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: 1: [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon.c 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: 2: [v2:172.21.15.46:3300/0,v1:172.21.15.46:6789/0] mon.b 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: fsmap 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: osdmap e113: 8 total, 8 up, 8 in 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: mgrmap e47: y(active, since 17s), standbys: x 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:56:49.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:49.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:49.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:49 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:56:50.134 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:56:50.134 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-30T09:56:50.135 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 5 2024-04-30T09:56:50.136 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:56:50.136 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:56:50.493 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:56:50.493 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:56:50.494 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:56:50.494 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi026", 2024-04-30T09:56:50.494 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:56:50.494 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "1/2 daemons upgraded", 2024-04-30T09:56:50.494 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mon daemons", 2024-04-30T09:56:50.494 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:56:50.494 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:56:50.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:50 smithi046 ceph-mon[65689]: from='client.34187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:50.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:50 smithi046 ceph-mon[65689]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:50.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:50 smithi046 ceph-mon[65689]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 956 B/s rd, 0 op/s 2024-04-30T09:56:50.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:50 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/3389823881' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:50.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:50 smithi026 ceph-mon[86310]: from='client.34187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:50.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:50 smithi026 ceph-mon[86310]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:50.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:50 smithi026 ceph-mon[86310]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 956 B/s rd, 0 op/s 2024-04-30T09:56:50.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:50 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/3389823881' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:50.941 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:50 smithi026 ceph-mon[80969]: from='client.34187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:50.941 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:50 smithi026 ceph-mon[80969]: from='client.44139 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:50.942 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:50 smithi026 ceph-mon[80969]: pgmap v13: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 956 B/s rd, 0 op/s 2024-04-30T09:56:50.942 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:50 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/3389823881' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:56:51.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:51 smithi026 ceph-mon[86310]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:51.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:51 smithi026 ceph-mon[86310]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:51.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:51 smithi026 ceph-mon[80969]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:51.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:51 smithi026 ceph-mon[80969]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:51.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:51 smithi046 ceph-mon[65689]: from='client.44145 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:51.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:51 smithi046 ceph-mon[65689]: from='client.44157 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:56:51.980 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:51 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T09:56:51.889+0000 7f3e7839e700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-30T09:56:52.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:52 smithi026 ceph-mon[86310]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 956 B/s rd, 0 op/s 2024-04-30T09:56:52.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:52 smithi026 ceph-mon[80969]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 956 B/s rd, 0 op/s 2024-04-30T09:56:52.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:52 smithi046 ceph-mon[65689]: pgmap v14: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 956 B/s rd, 0 op/s 2024-04-30T09:56:53.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:53 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:53.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:53 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:53.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:53 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:53.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:53 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:53.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:53 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:53.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:53 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:53.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:53 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:56:53.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:53 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:53.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:53 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:54.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:54 smithi046 ceph-mon[65689]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:54.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:54 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:54.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:54 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:54.768 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:54 smithi026 ceph-mon[86310]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:54.768 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:54 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:54.768 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:54 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:54.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:54 smithi026 ceph-mon[80969]: pgmap v15: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:54.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:54 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:54.768 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:54 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:56.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:56 smithi026 ceph-mon[80969]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:56.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:56 smithi026 ceph-mon[86310]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:56.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:56:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:56:56] "GET /metrics HTTP/1.1" 200 37996 "" "Prometheus/2.43.0" 2024-04-30T09:56:56.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:56 smithi046 ceph-mon[65689]: pgmap v16: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:56:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:56:58.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:56:58.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: pgmap v17: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:56:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: pgmap v18: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1011 B/s rd, 0 op/s 2024-04-30T09:56:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: pgmap v19: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:56:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:56:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:56:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:59.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[86310]: Reconfiguring mon.a (monmap changed)... 2024-04-30T09:56:59.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[86310]: Reconfiguring daemon mon.a on smithi026 2024-04-30T09:56:59.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:59.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:59.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:56:59.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:56:59.923 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:59.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[80969]: Reconfiguring mon.a (monmap changed)... 2024-04-30T09:56:59.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[80969]: Reconfiguring daemon mon.a on smithi026 2024-04-30T09:56:59.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:59.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:59.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:56:59.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:56:59.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:56:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:56:59.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:59 smithi046 ceph-mon[65689]: Reconfiguring mon.a (monmap changed)... 2024-04-30T09:56:59.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:59 smithi046 ceph-mon[65689]: Reconfiguring daemon mon.a on smithi026 2024-04-30T09:56:59.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:59.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:56:59.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:56:59.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:56:59.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:56:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:00.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:00 smithi026 ceph-mon[80969]: Reconfiguring mgr.y (monmap changed)... 2024-04-30T09:57:00.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:00 smithi026 ceph-mon[80969]: Reconfiguring daemon mgr.y on smithi026 2024-04-30T09:57:00.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:00 smithi026 ceph-mon[80969]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:00.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:00 smithi026 ceph-mon[86310]: Reconfiguring mgr.y (monmap changed)... 2024-04-30T09:57:00.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:00 smithi026 ceph-mon[86310]: Reconfiguring daemon mgr.y on smithi026 2024-04-30T09:57:00.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:00 smithi026 ceph-mon[86310]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:00.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:00 smithi046 ceph-mon[65689]: Reconfiguring mgr.y (monmap changed)... 2024-04-30T09:57:00.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:00 smithi046 ceph-mon[65689]: Reconfiguring daemon mgr.y on smithi026 2024-04-30T09:57:00.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:00 smithi046 ceph-mon[65689]: pgmap v20: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:01.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:01.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:01.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[86310]: Reconfiguring mon.c (monmap changed)... 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[86310]: Reconfiguring daemon mon.c on smithi026 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[86310]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[80969]: Reconfiguring mon.c (monmap changed)... 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:57:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:01.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[80969]: Reconfiguring daemon mon.c on smithi026 2024-04-30T09:57:01.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:01 smithi026 ceph-mon[80969]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-30T09:57:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:01 smithi046 ceph-mon[65689]: Reconfiguring mon.c (monmap changed)... 2024-04-30T09:57:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:57:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:57:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:01.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:01 smithi046 ceph-mon[65689]: Reconfiguring daemon mon.c on smithi026 2024-04-30T09:57:01.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:01 smithi046 ceph-mon[65689]: pgmap v21: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-30T09:57:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:02 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:02 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:02 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:02 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-30T09:57:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:02 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:03.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:03.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:03.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:03.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-30T09:57:03.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:03.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:03.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:03.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:03.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-30T09:57:03.314 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:02 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:03 smithi046 ceph-mon[65689]: Reconfiguring osd.0 (monmap changed)... 2024-04-30T09:57:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:03 smithi046 ceph-mon[65689]: Reconfiguring daemon osd.0 on smithi026 2024-04-30T09:57:04.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:03 smithi046 ceph-mon[65689]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:04.329 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:03 smithi026 ceph-mon[86310]: Reconfiguring osd.0 (monmap changed)... 2024-04-30T09:57:04.329 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:03 smithi026 ceph-mon[86310]: Reconfiguring daemon osd.0 on smithi026 2024-04-30T09:57:04.329 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:03 smithi026 ceph-mon[86310]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:04.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:03 smithi026 ceph-mon[80969]: Reconfiguring osd.0 (monmap changed)... 2024-04-30T09:57:04.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:03 smithi026 ceph-mon[80969]: Reconfiguring daemon osd.0 on smithi026 2024-04-30T09:57:04.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:03 smithi026 ceph-mon[80969]: pgmap v22: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:05.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[86310]: Reconfiguring osd.1 (monmap changed)... 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[86310]: Reconfiguring daemon osd.1 on smithi026 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[80969]: Reconfiguring osd.1 (monmap changed)... 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-30T09:57:05.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:05.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:05 smithi026 ceph-mon[80969]: Reconfiguring daemon osd.1 on smithi026 2024-04-30T09:57:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:05 smithi046 ceph-mon[65689]: Reconfiguring osd.1 (monmap changed)... 2024-04-30T09:57:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-30T09:57:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:05 smithi046 ceph-mon[65689]: Reconfiguring daemon osd.1 on smithi026 2024-04-30T09:57:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:57:06] "GET /metrics HTTP/1.1" 200 38015 "" "Prometheus/2.43.0" 2024-04-30T09:57:06.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[86310]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-30T09:57:06.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:06.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:06.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-30T09:57:06.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:06.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[80969]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-30T09:57:06.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:06.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:06.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-30T09:57:06.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:06 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:06.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:06 smithi046 ceph-mon[65689]: pgmap v23: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 630 B/s rd, 0 op/s 2024-04-30T09:57:06.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:06 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:06.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:06 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:06.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:06 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-30T09:57:06.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:06 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:07.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:07 smithi026 ceph-mon[86310]: Reconfiguring osd.2 (monmap changed)... 2024-04-30T09:57:07.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:07 smithi026 ceph-mon[86310]: Reconfiguring daemon osd.2 on smithi026 2024-04-30T09:57:07.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:07 smithi026 ceph-mon[80969]: Reconfiguring osd.2 (monmap changed)... 2024-04-30T09:57:07.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:07 smithi026 ceph-mon[80969]: Reconfiguring daemon osd.2 on smithi026 2024-04-30T09:57:07.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:07 smithi046 ceph-mon[65689]: Reconfiguring osd.2 (monmap changed)... 2024-04-30T09:57:07.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:07 smithi046 ceph-mon[65689]: Reconfiguring daemon osd.2 on smithi026 2024-04-30T09:57:08.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[86310]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:57:08.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:08.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:08.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-30T09:57:08.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[80969]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:57:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-30T09:57:08.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:08.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:08 smithi046 ceph-mon[65689]: pgmap v24: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:57:08.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:08.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:08.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-30T09:57:08.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:09.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:09 smithi026 ceph-mon[86310]: Reconfiguring osd.3 (monmap changed)... 2024-04-30T09:57:09.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:09 smithi026 ceph-mon[86310]: Reconfiguring daemon osd.3 on smithi026 2024-04-30T09:57:09.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:09 smithi026 ceph-mon[80969]: Reconfiguring osd.3 (monmap changed)... 2024-04-30T09:57:09.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:09 smithi026 ceph-mon[80969]: Reconfiguring daemon osd.3 on smithi026 2024-04-30T09:57:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:09 smithi046 ceph-mon[65689]: Reconfiguring osd.3 (monmap changed)... 2024-04-30T09:57:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:09 smithi046 ceph-mon[65689]: Reconfiguring daemon osd.3 on smithi026 2024-04-30T09:57:10.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[86310]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:10.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:10.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:10.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:57:10.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:10.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[80969]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:10.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:10.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:10.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:57:10.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:10 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:10.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:10 smithi046 ceph-mon[65689]: pgmap v25: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:10.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:10 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:10.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:10 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:10.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:10 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:57:10.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:10 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:11.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:11 smithi026 ceph-mon[86310]: Reconfiguring rgw.foo.smithi026.igccwv (monmap changed)... 2024-04-30T09:57:11.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:11 smithi026 ceph-mon[86310]: Reconfiguring daemon rgw.foo.smithi026.igccwv on smithi026 2024-04-30T09:57:11.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:11 smithi026 ceph-mon[80969]: Reconfiguring rgw.foo.smithi026.igccwv (monmap changed)... 2024-04-30T09:57:11.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:11 smithi026 ceph-mon[80969]: Reconfiguring daemon rgw.foo.smithi026.igccwv on smithi026 2024-04-30T09:57:11.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:11 smithi046 ceph-mon[65689]: Reconfiguring rgw.foo.smithi026.igccwv (monmap changed)... 2024-04-30T09:57:11.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:11 smithi046 ceph-mon[65689]: Reconfiguring daemon rgw.foo.smithi026.igccwv on smithi026 2024-04-30T09:57:12.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:12 smithi046 ceph-mon[65689]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:12.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:12 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:12.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:12 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:12.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:12 smithi046 ceph-mon[65689]: Reconfiguring mon.b (monmap changed)... 2024-04-30T09:57:12.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:12 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:57:12.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:12 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:57:12.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:12 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:12.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:12 smithi046 ceph-mon[65689]: Reconfiguring daemon mon.b on smithi046 2024-04-30T09:57:12.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:12 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:13.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[86310]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:13.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[86310]: Reconfiguring mon.b (monmap changed)... 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[86310]: Reconfiguring daemon mon.b on smithi046 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[80969]: pgmap v26: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:13.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[80969]: Reconfiguring mon.b (monmap changed)... 2024-04-30T09:57:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-30T09:57:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-30T09:57:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[80969]: Reconfiguring daemon mon.b on smithi046 2024-04-30T09:57:13.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:12 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:14 smithi046 ceph-mon[65689]: Reconfiguring mgr.x (monmap changed)... 2024-04-30T09:57:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:57:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:57:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:14 smithi046 ceph-mon[65689]: Reconfiguring daemon mgr.x on smithi046 2024-04-30T09:57:14.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[86310]: Reconfiguring mgr.x (monmap changed)... 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[86310]: Reconfiguring daemon mgr.x on smithi046 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[80969]: Reconfiguring mgr.x (monmap changed)... 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-30T09:57:14.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:14.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:14 smithi026 ceph-mon[80969]: Reconfiguring daemon mgr.x on smithi046 2024-04-30T09:57:15.576 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:15 smithi046 ceph-mon[65689]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:15.576 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:15 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:15.576 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:15 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:15.576 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:15 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-30T09:57:15.576 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:15 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:15.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[86310]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:15.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:15.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:15.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-30T09:57:15.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:15.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[80969]: pgmap v27: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:15.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:15.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:15.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-30T09:57:15.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:15 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:16.411 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:16 smithi046 ceph-mon[65689]: Reconfiguring osd.4 (monmap changed)... 2024-04-30T09:57:16.411 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:16 smithi046 ceph-mon[65689]: Reconfiguring daemon osd.4 on smithi046 2024-04-30T09:57:16.412 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:16.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:16 smithi026 ceph-mon[86310]: Reconfiguring osd.4 (monmap changed)... 2024-04-30T09:57:16.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:16 smithi026 ceph-mon[86310]: Reconfiguring daemon osd.4 on smithi046 2024-04-30T09:57:16.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:16.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:57:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:57:16] "GET /metrics HTTP/1.1" 200 38013 "" "Prometheus/2.43.0" 2024-04-30T09:57:16.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:16 smithi026 ceph-mon[80969]: Reconfiguring osd.4 (monmap changed)... 2024-04-30T09:57:16.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:16 smithi026 ceph-mon[80969]: Reconfiguring daemon osd.4 on smithi046 2024-04-30T09:57:16.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:17.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:17 smithi046 ceph-mon[65689]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:17.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:17 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:17.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:17 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:17.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:17 smithi046 ceph-mon[65689]: Reconfiguring osd.5 (monmap changed)... 2024-04-30T09:57:17.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:17 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-30T09:57:17.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:17 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[86310]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[86310]: Reconfiguring osd.5 (monmap changed)... 2024-04-30T09:57:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-30T09:57:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:17.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[80969]: pgmap v28: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:17.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:17.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:17.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[80969]: Reconfiguring osd.5 (monmap changed)... 2024-04-30T09:57:17.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-30T09:57:17.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:17 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:18.858 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:18 smithi046 ceph-mon[65689]: Reconfiguring daemon osd.5 on smithi046 2024-04-30T09:57:18.858 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:18.858 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:18.859 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-30T09:57:18.859 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:18.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[86310]: Reconfiguring daemon osd.5 on smithi046 2024-04-30T09:57:18.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:18.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:18.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-30T09:57:18.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:18.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[80969]: Reconfiguring daemon osd.5 on smithi046 2024-04-30T09:57:18.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:18.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:18.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-30T09:57:18.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:19.725 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:19 smithi046 ceph-mon[65689]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:19.725 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:19 smithi046 ceph-mon[65689]: Reconfiguring osd.6 (monmap changed)... 2024-04-30T09:57:19.725 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:19 smithi046 ceph-mon[65689]: Reconfiguring daemon osd.6 on smithi046 2024-04-30T09:57:19.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:19 smithi026 ceph-mon[86310]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:19.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:19 smithi026 ceph-mon[86310]: Reconfiguring osd.6 (monmap changed)... 2024-04-30T09:57:19.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:19 smithi026 ceph-mon[86310]: Reconfiguring daemon osd.6 on smithi046 2024-04-30T09:57:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:19 smithi026 ceph-mon[80969]: pgmap v29: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:19 smithi026 ceph-mon[80969]: Reconfiguring osd.6 (monmap changed)... 2024-04-30T09:57:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:19 smithi026 ceph-mon[80969]: Reconfiguring daemon osd.6 on smithi046 2024-04-30T09:57:20.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[86310]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:20.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:20.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:20.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[86310]: Reconfiguring osd.7 (monmap changed)... 2024-04-30T09:57:20.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-30T09:57:20.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:20.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[86310]: Reconfiguring daemon osd.7 on smithi046 2024-04-30T09:57:20.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[80969]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:20.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:20.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:20.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[80969]: Reconfiguring osd.7 (monmap changed)... 2024-04-30T09:57:20.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-30T09:57:20.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:20.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:20 smithi026 ceph-mon[80969]: Reconfiguring daemon osd.7 on smithi046 2024-04-30T09:57:20.880 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:57:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:20 smithi046 ceph-mon[65689]: pgmap v30: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:20 smithi046 ceph-mon[65689]: Reconfiguring osd.7 (monmap changed)... 2024-04-30T09:57:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-30T09:57:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:20 smithi046 ceph-mon[65689]: Reconfiguring daemon osd.7 on smithi046 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) 28s ago 25m 26.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (3m) 46s ago 25m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 28s ago 24m 62.0M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (4m) 46s ago 30m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (14m) 28s ago 32m 496M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (59s) 28s ago 32m 40.3M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (110s) 46s ago 30m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (35s) 28s ago 30m 22.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T09:57:21.562 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (4m) 28s ago 26m 15.7M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (4m) 46s ago 26m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (30m) 28s ago 30m 54.0M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (29m) 28s ago 29m 54.4M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (29m) 28s ago 29m 53.6M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (28m) 28s ago 28m 52.9M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (28m) 46s ago 28m 58.3M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (28m) 46s ago 28m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (27m) 46s ago 27m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (27m) 46s ago 27m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (3m) 46s ago 26m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (25m) 28s ago 25m 76.9M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:57:21.563 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (25m) 46s ago 25m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:57:21.979 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:57:21.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:57:21.980 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:57:21.980 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-30T09:57:21.980 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 5 2024-04-30T09:57:21.980 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:57:21.980 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) mon on host(s) smithi026", 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout: "mon" 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "2/2 daemons upgraded", 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading mon daemons", 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:57:22.333 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:57:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[86310]: from='client.44163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[86310]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[86310]: Reconfiguring rgw.foo.smithi046.wmzons (monmap changed)... 2024-04-30T09:57:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:57:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[86310]: Reconfiguring daemon rgw.foo.smithi046.wmzons on smithi046 2024-04-30T09:57:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/1874469525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[80969]: from='client.44163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[80969]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[80969]: Reconfiguring rgw.foo.smithi046.wmzons (monmap changed)... 2024-04-30T09:57:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:57:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[80969]: Reconfiguring daemon rgw.foo.smithi046.wmzons on smithi046 2024-04-30T09:57:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:22 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/1874469525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:22 smithi046 ceph-mon[65689]: from='client.44163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:22 smithi046 ceph-mon[65689]: from='client.44169 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:22 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:22.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:22 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:22.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:22 smithi046 ceph-mon[65689]: Reconfiguring rgw.foo.smithi046.wmzons (monmap changed)... 2024-04-30T09:57:22.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:22 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T09:57:22.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:22 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:22.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:22 smithi046 ceph-mon[65689]: Reconfiguring daemon rgw.foo.smithi046.wmzons on smithi046 2024-04-30T09:57:22.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:22 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/1874469525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:23.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='client.44175 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:23.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:23.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-30T09:57:23.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.739 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T09:57:23.740 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:57:23.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T09:57:23.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:57:23.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:57:23.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T09:57:23.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:57:23.741 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:57:23.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:57:23.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:57:23.742 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:57:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='client.44175 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: pgmap v31: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='client.44175 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:23.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='client.54127 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:57:23.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:57:23.955 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T09:57:23.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:57:23.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:57:23.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:57:23.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:57:23.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:57:23.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:57:23.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mon 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all crash 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mds 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nfs 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all node-exporter 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all prometheus 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all alertmanager 2024-04-30T09:57:24.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all grafana 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all loki 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all promtail 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Finalizing container_image settings 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: Upgrade: Complete! 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:57:24.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mon 2024-04-30T09:57:24.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all crash 2024-04-30T09:57:24.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mds 2024-04-30T09:57:24.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T09:57:24.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T09:57:24.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T09:57:24.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all nfs 2024-04-30T09:57:24.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:57:24.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all node-exporter 2024-04-30T09:57:24.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all prometheus 2024-04-30T09:57:24.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all alertmanager 2024-04-30T09:57:24.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all grafana 2024-04-30T09:57:24.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all loki 2024-04-30T09:57:24.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all promtail 2024-04-30T09:57:24.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Finalizing container_image settings 2024-04-30T09:57:24.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: Upgrade: Complete! 2024-04-30T09:57:24.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:24.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:57:24.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:57:24.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:24.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:57:24.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mon 2024-04-30T09:57:24.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all crash 2024-04-30T09:57:24.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mds 2024-04-30T09:57:24.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T09:57:24.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T09:57:24.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T09:57:24.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all nfs 2024-04-30T09:57:24.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:57:24.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all node-exporter 2024-04-30T09:57:24.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all prometheus 2024-04-30T09:57:24.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all alertmanager 2024-04-30T09:57:24.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all grafana 2024-04-30T09:57:24.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all loki 2024-04-30T09:57:24.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all promtail 2024-04-30T09:57:24.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Finalizing container_image settings 2024-04-30T09:57:24.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: Upgrade: Complete! 2024-04-30T09:57:24.953 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:24.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:57:24.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:24.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:57:24.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:57:24.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:57:24.955 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:25.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[86310]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:25.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[86310]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-30T09:57:25.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[86310]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[86310]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 0 op/s 2024-04-30T09:57:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[86310]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:57:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[80969]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[80969]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-30T09:57:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[80969]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[80969]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 0 op/s 2024-04-30T09:57:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:25 smithi026 ceph-mon[80969]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:57:25.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:25 smithi046 ceph-mon[65689]: pgmap v32: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:25.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:25 smithi046 ceph-mon[65689]: pgmap v33: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 991 B/s rd, 0 op/s 2024-04-30T09:57:25.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:25 smithi046 ceph-mon[65689]: pgmap v34: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:25.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:25 smithi046 ceph-mon[65689]: pgmap v35: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 790 B/s rd, 0 op/s 2024-04-30T09:57:25.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:25 smithi046 ceph-mon[65689]: pgmap v36: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:57:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:57:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:57:26] "GET /metrics HTTP/1.1" 200 38013 "" "Prometheus/2.43.0" 2024-04-30T09:57:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:26 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:26 smithi026 ceph-mon[86310]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:57:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:26 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:26 smithi026 ceph-mon[80969]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:57:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:26 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:57:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:26 smithi046 ceph-mon[65689]: pgmap v37: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:57:29.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:29 smithi026 ceph-mon[86310]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:57:29.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:29 smithi026 ceph-mon[80969]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:57:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:29 smithi046 ceph-mon[65689]: pgmap v38: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:57:31.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:31 smithi026 ceph-mon[86310]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-04-30T09:57:31.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:31.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:31 smithi026 ceph-mon[80969]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-04-30T09:57:31.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:31 smithi046 ceph-mon[65689]: pgmap v39: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-04-30T09:57:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:33.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:33 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:33.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:33 smithi026 ceph-mon[86310]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:33.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:33 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:33.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:33 smithi026 ceph-mon[80969]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:33 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:33 smithi046 ceph-mon[65689]: pgmap v40: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:35.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:35 smithi026 ceph-mon[86310]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:57:35.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:35 smithi026 ceph-mon[80969]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:57:35.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:35 smithi046 ceph-mon[65689]: pgmap v41: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:57:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:57:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:57:36] "GET /metrics HTTP/1.1" 200 38004 "" "Prometheus/2.43.0" 2024-04-30T09:57:37.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:37 smithi026 ceph-mon[86310]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:37 smithi026 ceph-mon[80969]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:37.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:37 smithi046 ceph-mon[65689]: pgmap v42: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:39.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:39 smithi026 ceph-mon[80969]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:39.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:39 smithi026 ceph-mon[86310]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:39 smithi046 ceph-mon[65689]: pgmap v43: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:41.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:41 smithi026 ceph-mon[86310]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:41.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:41 smithi026 ceph-mon[80969]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:41.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:41 smithi046 ceph-mon[65689]: pgmap v44: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:43.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:43 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:43.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:43 smithi026 ceph-mon[86310]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:43 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:43 smithi026 ceph-mon[80969]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:43 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:43 smithi046 ceph-mon[65689]: pgmap v45: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:45.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:45 smithi026 ceph-mon[86310]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:45.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:45 smithi026 ceph-mon[80969]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:45.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:45 smithi046 ceph-mon[65689]: pgmap v46: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:46.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:46.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:57:46.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:57:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:57:46] "GET /metrics HTTP/1.1" 200 38009 "" "Prometheus/2.43.0" 2024-04-30T09:57:47.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:47 smithi026 ceph-mon[80969]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:47.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:47 smithi026 ceph-mon[86310]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:47 smithi046 ceph-mon[65689]: pgmap v47: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:49.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:49 smithi026 ceph-mon[86310]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:49.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:49 smithi026 ceph-mon[80969]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:49 smithi046 ceph-mon[65689]: pgmap v48: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:51.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:51 smithi026 ceph-mon[86310]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:51 smithi026 ceph-mon[80969]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:51 smithi046 ceph-mon[65689]: pgmap v49: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:53 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:53.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:53 smithi026 ceph-mon[86310]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:53 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:53.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:53 smithi026 ceph-mon[80969]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:53 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:57:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:53 smithi046 ceph-mon[65689]: pgmap v50: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:53.505 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T09:57:54.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:54 smithi026 ceph-mon[86310]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:54.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:54 smithi026 ceph-mon[80969]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:54 smithi046 ceph-mon[65689]: from='client.44193 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:55.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:55 smithi026 ceph-mon[86310]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:55.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:55 smithi026 ceph-mon[80969]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:55 smithi046 ceph-mon[65689]: pgmap v51: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:55.539 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:57:55.539 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (4m) 62s ago 26m 26.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:57:55.539 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (3m) 80s ago 26m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:57:55.539 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 62s ago 25m 62.0M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:57:55.539 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (5m) 80s ago 31m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:57:55.539 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (15m) 62s ago 32m 496M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:57:55.539 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (93s) 62s ago 32m 40.3M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T09:57:55.539 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (2m) 80s ago 31m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (69s) 62s ago 31m 22.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (4m) 62s ago 27m 15.7M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (4m) 80s ago 27m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (30m) 62s ago 30m 54.0M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (30m) 62s ago 30m 54.4M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (29m) 62s ago 29m 53.6M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (29m) 62s ago 29m 52.9M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (29m) 80s ago 29m 58.3M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (28m) 80s ago 28m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (28m) 80s ago 28m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (27m) 80s ago 27m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (4m) 80s ago 27m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (25m) 62s ago 25m 76.9M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:57:55.540 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (25m) 80s ago 25m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:57:56.104 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.mon | length == 1'"'"'' 2024-04-30T09:57:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:56 smithi046 ceph-mon[65689]: from='client.54139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:56.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:56 smithi026 ceph-mon[80969]: from='client.54139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:56.364 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:56 smithi026 ceph-mon[86310]: from='client.54139 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:57:56.655 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:57:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:57:56] "GET /metrics HTTP/1.1" 200 38009 "" "Prometheus/2.43.0" 2024-04-30T09:57:57.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:57 smithi026 ceph-mon[80969]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:57.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:57 smithi026 ceph-mon[86310]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:57 smithi046 ceph-mon[65689]: pgmap v52: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:57:58.261 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:57:58.910 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.mon | keys'"'"' | grep $sha1' 2024-04-30T09:57:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:59 smithi026 ceph-mon[86310]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:59.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:57:59 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2186649943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:59.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:59 smithi026 ceph-mon[80969]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:59.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:57:59 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2186649943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:57:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:59 smithi046 ceph-mon[65689]: pgmap v53: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:57:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:57:59 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2186649943' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:01.012 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)" 2024-04-30T09:58:01.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:01 smithi026 ceph-mon[80969]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:01.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:01.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:01 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2271008393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:01.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:01 smithi026 ceph-mon[86310]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:01.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:01.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:01 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2271008393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:01 smithi046 ceph-mon[65689]: pgmap v54: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:01 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2271008393' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:01.647 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 5'"'"'' 2024-04-30T09:58:03.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:03 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:03.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:03 smithi026 ceph-mon[80969]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:58:03.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:03 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:03.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:03 smithi026 ceph-mon[86310]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:58:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:03 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:03 smithi046 ceph-mon[65689]: pgmap v55: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:58:05.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:05 smithi026 ceph-mon[86310]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:05.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:05 smithi026 ceph-mon[86310]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:05.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:05 smithi026 ceph-mon[80969]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:05.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:05 smithi026 ceph-mon[80969]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:05 smithi046 ceph-mon[65689]: from='client.44217 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:05 smithi046 ceph-mon[65689]: pgmap v56: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:05.742 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:58:06.363 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade status' 2024-04-30T09:58:06.675 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:58:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:58:06] "GET /metrics HTTP/1.1" 200 38004 "" "Prometheus/2.43.0" 2024-04-30T09:58:07.373 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:07 smithi026 ceph-mon[86310]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:07.374 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:07 smithi026 ceph-mon[80969]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:07 smithi046 ceph-mon[65689]: pgmap v57: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:08.323 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:58:08.323 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-30T09:58:08.323 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-30T09:58:08.323 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-30T09:58:08.323 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:58:08.323 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-30T09:58:08.324 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-30T09:58:08.324 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:58:08.324 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:58:08.958 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T09:58:09.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:09 smithi026 ceph-mon[86310]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:58:09.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:09 smithi026 ceph-mon[86310]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:09.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:09 smithi026 ceph-mon[80969]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:58:09.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:09 smithi026 ceph-mon[80969]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:09 smithi046 ceph-mon[65689]: pgmap v58: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:58:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:09 smithi046 ceph-mon[65689]: from='client.54163 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:11.014 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:58:11.014 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:58:11.014 INFO:teuthology.orchestra.run.smithi026.stdout: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:58:11.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:11 smithi026 ceph-mon[86310]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:11.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:11 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2592683303' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:58:11.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:11 smithi026 ceph-mon[80969]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:11.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:11 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2592683303' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:58:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:11 smithi046 ceph-mon[65689]: pgmap v59: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:11 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2592683303' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:58:11.687 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2' 2024-04-30T09:58:13.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:13 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:13.303 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:13 smithi026 ceph-mon[86310]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:58:13.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:13 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:13.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:13 smithi026 ceph-mon[80969]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:58:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:13 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:13 smithi046 ceph-mon[65689]: pgmap v60: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:58:15.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:15 smithi026 ceph-mon[80969]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:15 smithi026 ceph-mon[80969]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:15.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:15 smithi026 ceph-mon[86310]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:15.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:15 smithi026 ceph-mon[86310]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:15 smithi046 ceph-mon[65689]: from='client.54175 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "osd", "limit": 2, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:15 smithi046 ceph-mon[65689]: pgmap v61: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:15.996 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:58:16.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:16.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:16.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:16.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:16.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:16.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:16.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:58:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:58:16] "GET /metrics HTTP/1.1" 200 38010 "" "Prometheus/2.43.0" 2024-04-30T09:58:16.749 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-30T09:58:17.329 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[86310]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:58:17.329 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[86310]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:17.329 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:17.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:58:17.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:17.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[80969]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:58:17.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[80969]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:17.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:17.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:58:17.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:17 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:17 smithi046 ceph-mon[65689]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:58:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:17 smithi046 ceph-mon[65689]: pgmap v62: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:17 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:17 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:58:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:17 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:18.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:18 smithi026 ceph-mon[86310]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-30T09:58:18.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:18 smithi026 ceph-mon[86310]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:18.353 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:18 smithi026 ceph-mon[86310]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:58:18.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:18 smithi026 ceph-mon[80969]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-30T09:58:18.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:18 smithi026 ceph-mon[80969]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:18.353 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:18 smithi026 ceph-mon[80969]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:58:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:18 smithi046 ceph-mon[65689]: pgmap v63: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s rd, 1 op/s 2024-04-30T09:58:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:18 smithi046 ceph-mon[65689]: pgmap v64: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:18.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:18 smithi046 ceph-mon[65689]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:58:19.104 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:58:19.801 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (4m) 86s ago 26m 26.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (4m) 104s ago 26m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 86s ago 25m 62.0M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (5m) 104s ago 31m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (15m) 86s ago 33m 496M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (118s) 86s ago 33m 40.3M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (2m) 104s ago 31m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (93s) 86s ago 31m 22.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (5m) 86s ago 27m 15.7M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (5m) 104s ago 27m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (31m) 86s ago 31m 54.0M 2993M 17.2.0 e1d6a67b021e 4e96218d7585 2024-04-30T09:58:19.802 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (30m) 86s ago 30m 54.4M 2993M 17.2.0 e1d6a67b021e 127bb0d673a1 2024-04-30T09:58:19.803 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (30m) 86s ago 30m 53.6M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:58:19.803 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (29m) 86s ago 29m 52.9M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:58:19.803 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (29m) 104s ago 29m 58.3M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:58:19.803 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (29m) 104s ago 29m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:58:19.803 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (28m) 104s ago 28m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:58:19.803 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (28m) 104s ago 28m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:58:19.803 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (4m) 104s ago 27m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:58:19.803 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (26m) 86s ago 26m 76.9M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:58:19.804 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (25m) 104s ago 25m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:58:20.230 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:58:20.231 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:58:20.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:58:20.231 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:58:20.231 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 10, 2024-04-30T09:58:20.231 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 5 2024-04-30T09:58:20.231 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:58:20.231 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-30T09:58:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: pgmap v65: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='client.54181 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='client.54187 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:20.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-30T09:58:20.596 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:58:20.596 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:58:20.596 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:58:20.596 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (2 remaining).", 2024-04-30T09:58:20.596 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:58:20.596 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "0/8 daemons upgraded", 2024-04-30T09:58:20.597 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-30T09:58:20.597 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:58:20.597 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:58:21.406 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[86310]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:21.406 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[86310]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:58:21.406 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[86310]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:58:21.406 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mgr 2024-04-30T09:58:21.406 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mon 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all crash 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[86310]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[86310]: Upgrade: osd.0 is safe to restart 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/4223535978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[80969]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[80969]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[80969]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mgr 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mon 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all crash 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[80969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-30T09:58:21.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[80969]: Upgrade: osd.0 is safe to restart 2024-04-30T09:58:21.408 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:21 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/4223535978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:21 smithi046 ceph-mon[65689]: from='client.54193 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:21 smithi046 ceph-mon[65689]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:58:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:21 smithi046 ceph-mon[65689]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:58:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:21 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mgr 2024-04-30T09:58:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:21 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mon 2024-04-30T09:58:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:21 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all crash 2024-04-30T09:58:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:21 smithi046 ceph-mon[65689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-30T09:58:21.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:21 smithi046 ceph-mon[65689]: Upgrade: osd.0 is safe to restart 2024-04-30T09:58:21.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:21 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/4223535978' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[86310]: from='client.44265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[86310]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[86310]: Upgrade: Updating osd.0 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[86310]: Deploying daemon osd.0 on smithi026 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[80969]: from='client.44265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[80969]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s 2024-04-30T09:58:22.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[80969]: Upgrade: Updating osd.0 2024-04-30T09:58:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-30T09:58:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:22.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:22 smithi026 ceph-mon[80969]: Deploying daemon osd.0 on smithi026 2024-04-30T09:58:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:22 smithi046 ceph-mon[65689]: from='client.44265 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:22 smithi046 ceph-mon[65689]: pgmap v66: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 569 B/s rd, 0 op/s 2024-04-30T09:58:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:22 smithi046 ceph-mon[65689]: Upgrade: Updating osd.0 2024-04-30T09:58:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:22 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:22 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-30T09:58:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:22 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:22.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:22 smithi046 ceph-mon[65689]: Deploying daemon osd.0 on smithi026 2024-04-30T09:58:23.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:23 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:23.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:23 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:23 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:24 smithi046 ceph-mon[65689]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:24.942 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:24 smithi026 ceph-mon[80969]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:24.943 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:24 smithi026 ceph-mon[86310]: pgmap v67: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:24.943 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:24 smithi026 systemd[1]: Stopping Ceph osd.0 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:58:25.196 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:24 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[30006]: 2024-04-30T09:58:24.940+0000 7f6d9db4a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T09:58:25.196 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:24 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[30006]: 2024-04-30T09:58:24.940+0000 7f6d9db4a700 -1 osd.0 113 *** Got signal Terminated *** 2024-04-30T09:58:25.197 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:24 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[30006]: 2024-04-30T09:58:24.940+0000 7f6d9db4a700 -1 osd.0 113 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T09:58:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:25 smithi046 ceph-mon[65689]: osd.0 marked itself down and dead 2024-04-30T09:58:25.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:25 smithi026 ceph-mon[86310]: osd.0 marked itself down and dead 2024-04-30T09:58:25.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:25 smithi026 ceph-mon[80969]: osd.0 marked itself down and dead 2024-04-30T09:58:26.059 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:25 smithi026 podman[93176]: 2024-04-30 09:58:25.8012225 +0000 UTC m=+0.981553951 container died 4e96218d75856f98fc135302a4d3531b4603e8a7ce9c5353bbac93a0f0a05e6c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, vcs-type=git, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., architecture=x86_64, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, maintainer=Guillaume Abrioux , RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream) 2024-04-30T09:58:26.334 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:26 smithi026 podman[93176]: 2024-04-30 09:58:26.149491973 +0000 UTC m=+1.329823419 container cleanup 4e96218d75856f98fc135302a4d3531b4603e8a7ce9c5353bbac93a0f0a05e6c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , RELEASE=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, release=754, io.buildah.version=1.19.8, ceph=True, GIT_CLEAN=True, name=centos-stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, version=8, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container) 2024-04-30T09:58:26.334 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:26 smithi026 bash[93176]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0 2024-04-30T09:58:26.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:26 smithi026 ceph-mon[86310]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:26.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:26 smithi026 ceph-mon[86310]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T09:58:26.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:26 smithi026 ceph-mon[86310]: osdmap e114: 8 total, 7 up, 8 in 2024-04-30T09:58:26.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:58:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:58:26] "GET /metrics HTTP/1.1" 200 38010 "" "Prometheus/2.43.0" 2024-04-30T09:58:26.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:26 smithi026 ceph-mon[80969]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:26.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:26 smithi026 ceph-mon[80969]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T09:58:26.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:26 smithi026 ceph-mon[80969]: osdmap e114: 8 total, 7 up, 8 in 2024-04-30T09:58:26.697 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:26 smithi026 podman[93194]: 2024-04-30 09:58:26.334098375 +0000 UTC m=+0.527766723 container remove 4e96218d75856f98fc135302a4d3531b4603e8a7ce9c5353bbac93a0f0a05e6c (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0, com.redhat.component=centos-stream-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_CLEAN=True, architecture=x86_64, distribution-scope=public, GIT_BRANCH=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, io.buildah.version=1.19.8, RELEASE=HEAD, version=8, io.openshift.expose-services=, build-date=2022-05-03T08:36:31.336870, name=centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8) 2024-04-30T09:58:26.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:26 smithi046 ceph-mon[65689]: pgmap v68: 161 pgs: 161 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:26.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:26 smithi046 ceph-mon[65689]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T09:58:26.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:26 smithi046 ceph-mon[65689]: osdmap e114: 8 total, 7 up, 8 in 2024-04-30T09:58:27.104 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:26 smithi026 podman[93287]: 2024-04-30 09:58:26.696549992 +0000 UTC m=+0.018696386 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:58:27.105 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:26 smithi026 podman[93287]: 2024-04-30 09:58:26.801801252 +0000 UTC m=+0.123947614 container create 90f377cf67ded10dea7b7477dcd5adb00dd22ae5aef2b9f1eb4b8b5061062602 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T09:58:27.105 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:27 smithi026 podman[93287]: 2024-04-30 09:58:27.02143295 +0000 UTC m=+0.343579313 container init 90f377cf67ded10dea7b7477dcd5adb00dd22ae5aef2b9f1eb4b8b5061062602 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424) 2024-04-30T09:58:27.105 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:27 smithi026 podman[93287]: 2024-04-30 09:58:27.02520663 +0000 UTC m=+0.347352993 container start 90f377cf67ded10dea7b7477dcd5adb00dd22ae5aef2b9f1eb4b8b5061062602 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-30T09:58:27.446 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:27 smithi026 podman[93287]: 2024-04-30 09:58:27.104253459 +0000 UTC m=+0.426399825 container attach 90f377cf67ded10dea7b7477dcd5adb00dd22ae5aef2b9f1eb4b8b5061062602 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-30T09:58:27.446 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:27 smithi026 podman[93287]: 2024-04-30 09:58:27.347779749 +0000 UTC m=+0.669926106 container died 90f377cf67ded10dea7b7477dcd5adb00dd22ae5aef2b9f1eb4b8b5061062602 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-30T09:58:27.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:27 smithi046 ceph-mon[65689]: osdmap e115: 8 total, 7 up, 8 in 2024-04-30T09:58:27.908 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:27 smithi026 ceph-mon[86310]: osdmap e115: 8 total, 7 up, 8 in 2024-04-30T09:58:27.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:27 smithi026 ceph-mon[80969]: osdmap e115: 8 total, 7 up, 8 in 2024-04-30T09:58:28.197 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:27 smithi026 podman[93314]: 2024-04-30 09:58:27.908136107 +0000 UTC m=+0.554565281 container remove 90f377cf67ded10dea7b7477dcd5adb00dd22ae5aef2b9f1eb4b8b5061062602 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T09:58:28.197 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:27 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-30T09:58:28.197 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:27 smithi026 systemd[1]: Stopped Ceph osd.0 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:58:28.197 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:27 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.0.service: Consumed 15.510s CPU time. 2024-04-30T09:58:28.197 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:28 smithi026 systemd[1]: Starting Ceph osd.0 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:58:28.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:28 smithi026 ceph-mon[86310]: pgmap v71: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:28.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:28 smithi026 ceph-mon[86310]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:28.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:28 smithi026 ceph-mon[80969]: pgmap v71: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:28.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:28 smithi026 ceph-mon[80969]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:28.583 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:28 smithi026 podman[93429]: 2024-04-30 09:58:28.267669598 +0000 UTC m=+0.018929617 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:58:28.583 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:28 smithi026 podman[93429]: 2024-04-30 09:58:28.376370657 +0000 UTC m=+0.127630662 container create dc4e56446d0e13ba8f9189caf0bd8d0b1594fa08e3d1248aa71e71b088a2f26d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0) 2024-04-30T09:58:28.583 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:28 smithi026 podman[93429]: 2024-04-30 09:58:28.578655594 +0000 UTC m=+0.329915601 container init dc4e56446d0e13ba8f9189caf0bd8d0b1594fa08e3d1248aa71e71b088a2f26d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T09:58:28.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:28 smithi046 ceph-mon[65689]: pgmap v71: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:28.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:28 smithi046 ceph-mon[65689]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 20 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:28.946 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:28 smithi026 podman[93429]: 2024-04-30 09:58:28.582300533 +0000 UTC m=+0.333560538 container start dc4e56446d0e13ba8f9189caf0bd8d0b1594fa08e3d1248aa71e71b088a2f26d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2) 2024-04-30T09:58:28.946 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:28 smithi026 podman[93429]: 2024-04-30 09:58:28.678863185 +0000 UTC m=+0.430123204 container attach dc4e56446d0e13ba8f9189caf0bd8d0b1594fa08e3d1248aa71e71b088a2f26d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T09:58:29.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate[93445]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:58:29.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate[93445]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:58:29.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 bash[93429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:58:29.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 bash[93429]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:58:29.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 bash[93429]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-30T09:58:29.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 bash[93429]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-30T09:58:29.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 bash[93429]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-30T09:58:29.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 bash[93429]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 bash[93429]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 bash[93429]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 bash[93429]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate[93445]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate[93445]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate[93445]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate[93445]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate[93445]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate[93445]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate[93445]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-30T09:58:29.448 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 podman[93429]: 2024-04-30 09:58:29.099255496 +0000 UTC m=+0.850515492 container died dc4e56446d0e13ba8f9189caf0bd8d0b1594fa08e3d1248aa71e71b088a2f26d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True) 2024-04-30T09:58:30.117 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 podman[93579]: 2024-04-30 09:58:29.749660142 +0000 UTC m=+0.643608355 container remove dc4e56446d0e13ba8f9189caf0bd8d0b1594fa08e3d1248aa71e71b088a2f26d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-activate, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-04-30T09:58:30.117 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 podman[93645]: 2024-04-30 09:58:29.871822473 +0000 UTC m=+0.018606222 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:58:30.117 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:29 smithi026 podman[93645]: 2024-04-30 09:58:29.992223714 +0000 UTC m=+0.139007492 container create 860b97e5bf6e899e2c1a36a5d0034d00d963340fed4820f10bd2d989fc715d60 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-30T09:58:30.446 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:30 smithi026 podman[93645]: 2024-04-30 09:58:30.219728423 +0000 UTC m=+0.366512167 container init 860b97e5bf6e899e2c1a36a5d0034d00d963340fed4820f10bd2d989fc715d60 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5) 2024-04-30T09:58:30.446 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:30 smithi026 podman[93645]: 2024-04-30 09:58:30.223128798 +0000 UTC m=+0.369912534 container start 860b97e5bf6e899e2c1a36a5d0034d00d963340fed4820f10bd2d989fc715d60 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T09:58:30.446 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-osd[93664]: -- 172.21.15.26:0/3752317572 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55aaa9e07680 con 0x55aaa9dd0c00 2024-04-30T09:58:30.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:30 smithi026 bash[93645]: 860b97e5bf6e899e2c1a36a5d0034d00d963340fed4820f10bd2d989fc715d60 2024-04-30T09:58:30.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:30 smithi026 systemd[1]: Started Ceph osd.0 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:58:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[86310]: pgmap v72: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:30.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:30.773 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[80969]: pgmap v72: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:30.773 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:30.773 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:30.774 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:30.774 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:30.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:30 smithi046 ceph-mon[65689]: pgmap v72: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:30.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:30.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:30.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:30.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:31.196 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:30 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[93660]: 2024-04-30T09:58:30.771+0000 7f2771a387c0 -1 Falling back to public interface 2024-04-30T09:58:32.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:32.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:32.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:31 smithi026 ceph-mon[86310]: pgmap v73: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:32.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:32.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:32.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:31 smithi026 ceph-mon[80969]: pgmap v73: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:31 smithi046 ceph-mon[65689]: pgmap v73: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:33.054 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:32 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:33.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:32 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:32 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:33.787 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[93660]: 2024-04-30T09:58:33.511+0000 7f2771a387c0 -1 osd.0 0 read_superblock omap replica is missing. 2024-04-30T09:58:33.787 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[93660]: 2024-04-30T09:58:33.544+0000 7f2771a387c0 -1 osd.0 113 log_to_monitors true 2024-04-30T09:58:34.196 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 09:58:33 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[93660]: 2024-04-30T09:58:33.813+0000 7f2762e8f700 -1 osd.0 113 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:58:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:33 smithi026 ceph-mon[80969]: pgmap v74: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:34.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:33 smithi026 ceph-mon[80969]: from='osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-30T09:58:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:33 smithi026 ceph-mon[86310]: pgmap v74: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:34.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:33 smithi026 ceph-mon[86310]: from='osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-30T09:58:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:33 smithi046 ceph-mon[65689]: pgmap v74: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 99 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T09:58:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:33 smithi046 ceph-mon[65689]: from='osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-30T09:58:34.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:34 smithi026 ceph-mon[86310]: from='osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-30T09:58:34.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:34 smithi026 ceph-mon[86310]: osdmap e116: 8 total, 7 up, 8 in 2024-04-30T09:58:34.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:34 smithi026 ceph-mon[86310]: from='osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:58:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:34 smithi026 ceph-mon[80969]: from='osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-30T09:58:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:34 smithi026 ceph-mon[80969]: osdmap e116: 8 total, 7 up, 8 in 2024-04-30T09:58:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:34 smithi026 ceph-mon[80969]: from='osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:58:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:34 smithi046 ceph-mon[65689]: from='osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-30T09:58:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:34 smithi046 ceph-mon[65689]: osdmap e116: 8 total, 7 up, 8 in 2024-04-30T09:58:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:34 smithi046 ceph-mon[65689]: from='osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:58:36.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[80969]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[80969]: osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899] boot 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[80969]: osdmap e117: 8 total, 8 up, 8 in 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[80969]: pgmap v77: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[80969]: osdmap e118: 8 total, 8 up, 8 in 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[86310]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[86310]: osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899] boot 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[86310]: osdmap e117: 8 total, 8 up, 8 in 2024-04-30T09:58:36.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:58:36.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:36.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:36.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[86310]: pgmap v77: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-30T09:58:36.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:35 smithi026 ceph-mon[86310]: osdmap e118: 8 total, 8 up, 8 in 2024-04-30T09:58:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:35 smithi046 ceph-mon[65689]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T09:58:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:35 smithi046 ceph-mon[65689]: osd.0 [v2:172.21.15.26:6802/1580697899,v1:172.21.15.26:6803/1580697899] boot 2024-04-30T09:58:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:35 smithi046 ceph-mon[65689]: osdmap e117: 8 total, 8 up, 8 in 2024-04-30T09:58:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:35 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-30T09:58:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:35 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:35 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:35 smithi046 ceph-mon[65689]: pgmap v77: 161 pgs: 20 active+undersized+degraded, 38 active+undersized, 103 active+clean; 585 KiB data, 115 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 68/627 objects degraded (10.845%) 2024-04-30T09:58:36.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:35 smithi046 ceph-mon[65689]: osdmap e118: 8 total, 8 up, 8 in 2024-04-30T09:58:36.664 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:58:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:58:36] "GET /metrics HTTP/1.1" 200 38142 "" "Prometheus/2.43.0" 2024-04-30T09:58:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:37 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:37 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:37 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:37 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:37.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:37 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:37.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:37 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:38.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:38 smithi026 ceph-mon[86310]: pgmap v79: 161 pgs: 9 active+undersized+degraded, 12 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 31/627 objects degraded (4.944%) 2024-04-30T09:58:38.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:38 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:38 smithi026 ceph-mon[80969]: pgmap v79: 161 pgs: 9 active+undersized+degraded, 12 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 31/627 objects degraded (4.944%) 2024-04-30T09:58:38.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:38 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:38 smithi046 ceph-mon[65689]: pgmap v79: 161 pgs: 9 active+undersized+degraded, 12 active+undersized, 140 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 31/627 objects degraded (4.944%) 2024-04-30T09:58:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:38 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:39.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:39 smithi026 ceph-mon[86310]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded) 2024-04-30T09:58:39.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:39 smithi026 ceph-mon[80969]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded) 2024-04-30T09:58:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:39 smithi046 ceph-mon[65689]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 9 pgs degraded) 2024-04-30T09:58:40.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:40.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:40.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:40.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:40.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:58:40.514 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 901 B/s rd, 0 op/s 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:40.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:40.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:58:40.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 901 B/s rd, 0 op/s 2024-04-30T09:58:40.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:40.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:40.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:40.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:40.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:40.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:40.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:40 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-30T09:58:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: pgmap v80: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: pgmap v81: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 901 B/s rd, 0 op/s 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: pgmap v82: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:40 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-30T09:58:41.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:41 smithi046 ceph-mon[65689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-30T09:58:41.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:41 smithi046 ceph-mon[65689]: Upgrade: osd.1 is safe to restart 2024-04-30T09:58:41.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:41 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:41.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:41 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-30T09:58:41.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:41 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:41.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[80969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-30T09:58:41.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[80969]: Upgrade: osd.1 is safe to restart 2024-04-30T09:58:41.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:41.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-30T09:58:41.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:41.817 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[86310]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-30T09:58:41.817 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[86310]: Upgrade: osd.1 is safe to restart 2024-04-30T09:58:41.817 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:41.817 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-30T09:58:41.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:41 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:58:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:42 smithi046 ceph-mon[65689]: Upgrade: Updating osd.1 2024-04-30T09:58:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:42 smithi046 ceph-mon[65689]: Deploying daemon osd.1 on smithi026 2024-04-30T09:58:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:42 smithi046 ceph-mon[65689]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 793 B/s rd, 0 op/s 2024-04-30T09:58:42.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:42 smithi026 ceph-mon[86310]: Upgrade: Updating osd.1 2024-04-30T09:58:42.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:42 smithi026 ceph-mon[86310]: Deploying daemon osd.1 on smithi026 2024-04-30T09:58:42.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:42 smithi026 ceph-mon[86310]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 793 B/s rd, 0 op/s 2024-04-30T09:58:42.808 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:42 smithi026 ceph-mon[80969]: Upgrade: Updating osd.1 2024-04-30T09:58:42.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:42 smithi026 ceph-mon[80969]: Deploying daemon osd.1 on smithi026 2024-04-30T09:58:42.809 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:42 smithi026 ceph-mon[80969]: pgmap v83: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 793 B/s rd, 0 op/s 2024-04-30T09:58:43.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:43 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:43.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:43 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:43.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:43 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:44.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:44 smithi026 ceph-mon[80969]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T09:58:44.872 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:44 smithi026 ceph-mon[86310]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T09:58:44.872 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:44 smithi026 systemd[1]: Stopping Ceph osd.1 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:58:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:44 smithi046 ceph-mon[65689]: pgmap v84: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T09:58:45.196 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:44 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[32991]: 2024-04-30T09:58:44.869+0000 7fce74945700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T09:58:45.197 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:44 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[32991]: 2024-04-30T09:58:44.869+0000 7fce74945700 -1 osd.1 118 *** Got signal Terminated *** 2024-04-30T09:58:45.197 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:44 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[32991]: 2024-04-30T09:58:44.869+0000 7fce74945700 -1 osd.1 118 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T09:58:45.885 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:45 smithi046 ceph-mon[65689]: osd.1 marked itself down and dead 2024-04-30T09:58:45.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:45 smithi026 ceph-mon[86310]: osd.1 marked itself down and dead 2024-04-30T09:58:45.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:45 smithi026 ceph-mon[80969]: osd.1 marked itself down and dead 2024-04-30T09:58:46.222 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:45 smithi026 podman[97949]: 2024-04-30 09:58:45.963994222 +0000 UTC m=+1.230551500 container died 127bb0d673a1dc7d1e0b795691f394899a7984e75f03b878dfe67c44d8e7b2da (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, version=8, build-date=2022-05-03T08:36:31.336870, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=, architecture=x86_64, com.redhat.component=centos-stream-container, RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , com.redhat.license_terms=https://centos.org/legal/licensing-policy/, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, release=754, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.buildah.version=1.19.8, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_CLEAN=True) 2024-04-30T09:58:46.497 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:58:46] "GET /metrics HTTP/1.1" 200 38155 "" "Prometheus/2.43.0" 2024-04-30T09:58:46.497 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:46 smithi026 podman[97949]: 2024-04-30 09:58:46.311698784 +0000 UTC m=+1.578256050 container cleanup 127bb0d673a1dc7d1e0b795691f394899a7984e75f03b878dfe67c44d8e7b2da (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, version=8, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, name=centos-stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, ceph=True, io.openshift.expose-services=, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, release=754, GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-30T09:58:46.497 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:46 smithi026 bash[97949]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1 2024-04-30T09:58:46.943 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[86310]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 605 B/s rd, 0 op/s 2024-04-30T09:58:46.943 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[86310]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T09:58:46.943 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[86310]: osdmap e119: 8 total, 7 up, 8 in 2024-04-30T09:58:46.943 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:46.943 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:46.943 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:46.944 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:46 smithi026 podman[97966]: 2024-04-30 09:58:46.496396654 +0000 UTC m=+0.522615505 container remove 127bb0d673a1dc7d1e0b795691f394899a7984e75f03b878dfe67c44d8e7b2da (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1, io.openshift.tags=base centos centos-stream, io.buildah.version=1.19.8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, vendor=Red Hat, Inc., release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.expose-services=, maintainer=Guillaume Abrioux , name=centos-stream, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, version=8, GIT_CLEAN=True) 2024-04-30T09:58:46.944 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[80969]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 605 B/s rd, 0 op/s 2024-04-30T09:58:46.944 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[80969]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T09:58:46.944 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[80969]: osdmap e119: 8 total, 7 up, 8 in 2024-04-30T09:58:46.944 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:46.944 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:46.944 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:46 smithi046 ceph-mon[65689]: pgmap v85: 161 pgs: 161 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 605 B/s rd, 0 op/s 2024-04-30T09:58:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:46 smithi046 ceph-mon[65689]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T09:58:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:46 smithi046 ceph-mon[65689]: osdmap e119: 8 total, 7 up, 8 in 2024-04-30T09:58:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:58:46.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:47.195 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:46 smithi026 podman[98056]: 2024-04-30 09:58:46.846388693 +0000 UTC m=+0.018705779 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:58:47.195 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:46 smithi026 podman[98056]: 2024-04-30 09:58:46.947363779 +0000 UTC m=+0.119680857 container create cc1b813ffab8e4bffc505fd7ea9a63e5e1f84318f53aa00083bc3a87d32ade4d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=) 2024-04-30T09:58:47.195 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:47 smithi026 podman[98056]: 2024-04-30 09:58:47.191762793 +0000 UTC m=+0.364079865 container init cc1b813ffab8e4bffc505fd7ea9a63e5e1f84318f53aa00083bc3a87d32ade4d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-30T09:58:47.195 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:47 smithi026 podman[98056]: 2024-04-30 09:58:47.194917421 +0000 UTC m=+0.367234494 container start cc1b813ffab8e4bffc505fd7ea9a63e5e1f84318f53aa00083bc3a87d32ade4d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T09:58:47.446 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:47 smithi026 podman[98056]: 2024-04-30 09:58:47.300122745 +0000 UTC m=+0.472439831 container attach cc1b813ffab8e4bffc505fd7ea9a63e5e1f84318f53aa00083bc3a87d32ade4d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T09:58:47.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:47 smithi026 ceph-mon[86310]: osdmap e120: 8 total, 7 up, 8 in 2024-04-30T09:58:47.797 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:47 smithi026 podman[98056]: 2024-04-30 09:58:47.499681338 +0000 UTC m=+0.671998402 container died cc1b813ffab8e4bffc505fd7ea9a63e5e1f84318f53aa00083bc3a87d32ade4d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, GIT_BRANCH=HEAD) 2024-04-30T09:58:47.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:47 smithi026 ceph-mon[80969]: osdmap e120: 8 total, 7 up, 8 in 2024-04-30T09:58:47.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:47 smithi046 ceph-mon[65689]: osdmap e120: 8 total, 7 up, 8 in 2024-04-30T09:58:48.447 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 podman[98081]: 2024-04-30 09:58:48.070438835 +0000 UTC m=+0.565285824 container remove cc1b813ffab8e4bffc505fd7ea9a63e5e1f84318f53aa00083bc3a87d32ade4d (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, ceph=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-30T09:58:48.447 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-30T09:58:48.447 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 systemd[1]: Stopped Ceph osd.1 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:58:48.447 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.1.service: Consumed 16.369s CPU time. 2024-04-30T09:58:48.447 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 systemd[1]: Starting Ceph osd.1 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:58:48.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:48 smithi026 ceph-mon[86310]: pgmap v88: 161 pgs: 18 active+undersized+degraded, 1 stale+active+clean, 38 active+undersized, 104 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/627 objects degraded (12.121%) 2024-04-30T09:58:48.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:48 smithi026 ceph-mon[86310]: Health check failed: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:48.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:48 smithi026 ceph-mon[80969]: pgmap v88: 161 pgs: 18 active+undersized+degraded, 1 stale+active+clean, 38 active+undersized, 104 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/627 objects degraded (12.121%) 2024-04-30T09:58:48.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:48 smithi026 ceph-mon[80969]: Health check failed: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:48.851 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 podman[98184]: 2024-04-30 09:58:48.431218046 +0000 UTC m=+0.019763517 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:58:48.851 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 podman[98184]: 2024-04-30 09:58:48.555006046 +0000 UTC m=+0.143551517 container create 7c724df2e6852da7ce718705e3ae4951ecf680444e5b29350033c36d8a627bea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True) 2024-04-30T09:58:48.851 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 podman[98184]: 2024-04-30 09:58:48.766869094 +0000 UTC m=+0.355414587 container init 7c724df2e6852da7ce718705e3ae4951ecf680444e5b29350033c36d8a627bea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-30T09:58:48.851 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 podman[98184]: 2024-04-30 09:58:48.770321545 +0000 UTC m=+0.358867025 container start 7c724df2e6852da7ce718705e3ae4951ecf680444e5b29350033c36d8a627bea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-30T09:58:48.852 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:48 smithi026 podman[98184]: 2024-04-30 09:58:48.849406619 +0000 UTC m=+0.437952104 container attach 7c724df2e6852da7ce718705e3ae4951ecf680444e5b29350033c36d8a627bea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-04-30T09:58:48.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:48 smithi046 ceph-mon[65689]: pgmap v88: 161 pgs: 18 active+undersized+degraded, 1 stale+active+clean, 38 active+undersized, 104 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 76/627 objects degraded (12.121%) 2024-04-30T09:58:48.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:48 smithi046 ceph-mon[65689]: Health check failed: Degraded data redundancy: 76/627 objects degraded (12.121%), 18 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate[98199]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate[98199]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 bash[98184]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 bash[98184]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 bash[98184]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 bash[98184]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 bash[98184]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 bash[98184]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 bash[98184]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 bash[98184]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 bash[98184]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-30T09:58:49.697 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate[98199]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-30T09:58:49.698 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate[98199]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-30T09:58:49.698 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate[98199]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-30T09:58:49.698 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate[98199]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-30T09:58:49.698 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate[98199]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-30T09:58:49.698 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate[98199]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-30T09:58:49.698 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate[98199]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-30T09:58:49.698 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:49 smithi026 podman[98184]: 2024-04-30 09:58:49.299207453 +0000 UTC m=+0.887752924 container died 7c724df2e6852da7ce718705e3ae4951ecf680444e5b29350033c36d8a627bea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T09:58:50.355 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:50 smithi026 podman[98339]: 2024-04-30 09:58:50.104019736 +0000 UTC m=+0.793636963 container remove 7c724df2e6852da7ce718705e3ae4951ecf680444e5b29350033c36d8a627bea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-activate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2) 2024-04-30T09:58:50.355 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:50 smithi026 podman[98405]: 2024-04-30 09:58:50.22835652 +0000 UTC m=+0.018584579 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:58:50.355 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:50 smithi026 podman[98405]: 2024-04-30 09:58:50.354778997 +0000 UTC m=+0.145007036 container create 79002116badcecb2f5b65e41e21979c5ec18e5bec7b70e4858cb0d842e5a47a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-04-30T09:58:50.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:50 smithi026 ceph-mon[86310]: pgmap v89: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:50.691 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:50 smithi026 podman[98405]: 2024-04-30 09:58:50.565714575 +0000 UTC m=+0.355942615 container init 79002116badcecb2f5b65e41e21979c5ec18e5bec7b70e4858cb0d842e5a47a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1, GIT_CLEAN=True, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T09:58:50.691 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:50 smithi026 podman[98405]: 2024-04-30 09:58:50.568845871 +0000 UTC m=+0.359073911 container start 79002116badcecb2f5b65e41e21979c5ec18e5bec7b70e4858cb0d842e5a47a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-30T09:58:50.691 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:50 smithi026 ceph-osd[98423]: -- 172.21.15.26:0/3997905791 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558d80567680 con 0x558d80531400 2024-04-30T09:58:50.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:50 smithi026 ceph-mon[80969]: pgmap v89: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:50.946 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:50 smithi026 bash[98405]: 79002116badcecb2f5b65e41e21979c5ec18e5bec7b70e4858cb0d842e5a47a2 2024-04-30T09:58:50.946 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:50 smithi026 systemd[1]: Started Ceph osd.1 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:58:50.984 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:58:50.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:50 smithi046 ceph-mon[65689]: pgmap v89: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:51.446 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[98419]: 2024-04-30T09:58:51.108+0000 7f8cdf8df7c0 -1 Falling back to public interface 2024-04-30T09:58:51.716 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:58:51.716 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (4m) 16s ago 27m 25.7M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:58:51.716 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (4m) 2m ago 27m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:58:51.716 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 16s ago 26m 62.0M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:58:51.716 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (6m) 2m ago 31m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (16m) 16s ago 33m 505M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (2m) 16s ago 33m 46.6M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (3m) 2m ago 32m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (2m) 16s ago 32m 39.2M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (5m) 16s ago 28m 15.5M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (5m) 2m ago 27m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (21s) 16s ago 31m 15.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 starting - - - 2993M 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (30m) 16s ago 30m 54.0M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (30m) 16s ago 30m 53.5M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (30m) 2m ago 30m 58.3M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (29m) 2m ago 29m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (29m) 2m ago 29m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (28m) 2m ago 28m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:58:51.717 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (5m) 2m ago 28m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:58:51.718 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (26m) 16s ago 26m 77.3M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:58:51.718 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (26m) 2m ago 26m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:58:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[80969]: from='client.44280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[80969]: from='client.44286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:51.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[80969]: pgmap v90: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[86310]: from='client.44280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[86310]: from='client.44286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:51.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:51 smithi026 ceph-mon[86310]: pgmap v90: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:52.150 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:58:52.150 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:58:52.150 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T09:58:52.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:58:52.150 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:58:52.150 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:58:52.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:58:52.150 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 1 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 6 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:58:52.151 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:58:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:51 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:51 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:51 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:51 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:58:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:51 smithi046 ceph-mon[65689]: from='client.44280 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:51 smithi046 ceph-mon[65689]: from='client.44286 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:52.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:51 smithi046 ceph-mon[65689]: pgmap v90: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:52.528 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:58:52.528 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T09:58:52.528 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:58:52.528 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 2 daemons (0 remaining).", 2024-04-30T09:58:52.528 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:58:52.528 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "1/8 daemons upgraded", 2024-04-30T09:58:52.529 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-30T09:58:52.529 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:58:52.529 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:58:52.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:52 smithi026 ceph-mon[86310]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:52.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:52 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:52.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:52 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2903038970' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:52.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:52 smithi026 ceph-mon[86310]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:52.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:52 smithi026 ceph-mon[80969]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:52.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:52 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:52.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:52 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2903038970' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:52.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:52 smithi026 ceph-mon[80969]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:53.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:52 smithi046 ceph-mon[65689]: from='client.44292 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:53.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:52 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:58:53.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:52 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2903038970' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:58:53.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:52 smithi046 ceph-mon[65689]: from='client.44304 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:58:54.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:53 smithi026 ceph-mon[86310]: pgmap v91: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:54.141 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[98419]: 2024-04-30T09:58:53.826+0000 7f8cdf8df7c0 -1 osd.1 0 read_superblock omap replica is missing. 2024-04-30T09:58:54.141 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[98419]: 2024-04-30T09:58:53.853+0000 7f8cdf8df7c0 -1 osd.1 118 log_to_monitors true 2024-04-30T09:58:54.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:53 smithi026 ceph-mon[80969]: pgmap v91: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:54.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:53 smithi046 ceph-mon[65689]: pgmap v91: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 116 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:55.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:54 smithi026 ceph-mon[80969]: from='osd.1 [v2:172.21.15.26:6810/1092166723,v1:172.21.15.26:6811/1092166723]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:58:55.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:54 smithi026 ceph-mon[80969]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:58:55.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:54 smithi026 ceph-mon[86310]: from='osd.1 [v2:172.21.15.26:6810/1092166723,v1:172.21.15.26:6811/1092166723]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:58:55.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:54 smithi026 ceph-mon[86310]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:58:55.197 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 09:58:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[98419]: 2024-04-30T09:58:54.816+0000 7f8cd0d36700 -1 osd.1 118 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T09:58:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:54 smithi046 ceph-mon[65689]: from='osd.1 [v2:172.21.15.26:6810/1092166723,v1:172.21.15.26:6811/1092166723]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:58:55.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:54 smithi046 ceph-mon[65689]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-30T09:58:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[86310]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-30T09:58:56.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[86310]: osdmap e121: 8 total, 7 up, 8 in 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[86310]: from='osd.1 [v2:172.21.15.26:6810/1092166723,v1:172.21.15.26:6811/1092166723]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[86310]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[86310]: pgmap v93: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[80969]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[80969]: osdmap e121: 8 total, 7 up, 8 in 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[80969]: from='osd.1 [v2:172.21.15.26:6810/1092166723,v1:172.21.15.26:6811/1092166723]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[80969]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:56.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:55 smithi026 ceph-mon[80969]: pgmap v93: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:55 smithi046 ceph-mon[65689]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-30T09:58:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:55 smithi046 ceph-mon[65689]: osdmap e121: 8 total, 7 up, 8 in 2024-04-30T09:58:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:55 smithi046 ceph-mon[65689]: from='osd.1 [v2:172.21.15.26:6810/1092166723,v1:172.21.15.26:6811/1092166723]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:58:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:55 smithi046 ceph-mon[65689]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T09:58:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:55 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:55 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:56.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:55 smithi046 ceph-mon[65689]: pgmap v93: 161 pgs: 20 active+undersized+degraded, 43 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 83/627 objects degraded (13.238%) 2024-04-30T09:58:56.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:58:56] "GET /metrics HTTP/1.1" 200 38155 "" "Prometheus/2.43.0" 2024-04-30T09:58:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:56 smithi046 ceph-mon[65689]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T09:58:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:56 smithi046 ceph-mon[65689]: osd.1 [v2:172.21.15.26:6810/1092166723,v1:172.21.15.26:6811/1092166723] boot 2024-04-30T09:58:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:56 smithi046 ceph-mon[65689]: osdmap e122: 8 total, 8 up, 8 in 2024-04-30T09:58:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:56 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:58:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:56 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:56 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:56 smithi046 ceph-mon[65689]: osdmap e123: 8 total, 8 up, 8 in 2024-04-30T09:58:57.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[86310]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T09:58:57.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[86310]: osd.1 [v2:172.21.15.26:6810/1092166723,v1:172.21.15.26:6811/1092166723] boot 2024-04-30T09:58:57.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[86310]: osdmap e122: 8 total, 8 up, 8 in 2024-04-30T09:58:57.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:58:57.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:57.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:57.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[86310]: osdmap e123: 8 total, 8 up, 8 in 2024-04-30T09:58:57.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[80969]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T09:58:57.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[80969]: osd.1 [v2:172.21.15.26:6810/1092166723,v1:172.21.15.26:6811/1092166723] boot 2024-04-30T09:58:57.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[80969]: osdmap e122: 8 total, 8 up, 8 in 2024-04-30T09:58:57.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-30T09:58:57.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:57.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:58:57.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:56 smithi026 ceph-mon[80969]: osdmap e123: 8 total, 8 up, 8 in 2024-04-30T09:58:58.125 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:57 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 20 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:58.125 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:58:57 smithi026 ceph-mon[86310]: pgmap v96: 161 pgs: 37 peering, 7 active+undersized+degraded, 19 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 36/627 objects degraded (5.742%) 2024-04-30T09:58:58.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:57 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 20 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:58.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:58:57 smithi026 ceph-mon[80969]: pgmap v96: 161 pgs: 37 peering, 7 active+undersized+degraded, 19 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 36/627 objects degraded (5.742%) 2024-04-30T09:58:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:57 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 83/627 objects degraded (13.238%), 20 pgs degraded (PG_DEGRADED) 2024-04-30T09:58:58.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:58:57 smithi046 ceph-mon[65689]: pgmap v96: 161 pgs: 37 peering, 7 active+undersized+degraded, 19 active+undersized, 98 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 36/627 objects degraded (5.742%) 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: pgmap v97: 161 pgs: 37 peering, 6 active+undersized+degraded, 14 active+undersized, 104 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 30/627 objects degraded (4.785%) 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:59:00.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: pgmap v97: 161 pgs: 37 peering, 6 active+undersized+degraded, 14 active+undersized, 104 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 30/627 objects degraded (4.785%) 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:59:00.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:00.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.953 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:59:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:59:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:00.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: pgmap v97: 161 pgs: 37 peering, 6 active+undersized+degraded, 14 active+undersized, 104 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 30/627 objects degraded (4.785%) 2024-04-30T09:59:00.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T09:59:00.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:00.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:00.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:00.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:00.988 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:01.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: pgmap v98: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 965 B/s rd, 0 op/s 2024-04-30T09:59:01.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: pgmap v99: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:01.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:59:01.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: Upgrade: Finalizing container_image settings 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: Upgrade: Complete! 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: pgmap v100: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: pgmap v101: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: pgmap v102: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: pgmap v103: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[86310]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/627 objects degraded (4.785%), 6 pgs degraded) 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: pgmap v98: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 965 B/s rd, 0 op/s 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: pgmap v99: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:01.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: Upgrade: Finalizing container_image settings 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: Upgrade: Complete! 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: pgmap v100: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: pgmap v101: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: pgmap v102: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: pgmap v103: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:01.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:01 smithi026 ceph-mon[80969]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/627 objects degraded (4.785%), 6 pgs degraded) 2024-04-30T09:59:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: pgmap v98: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 965 B/s rd, 0 op/s 2024-04-30T09:59:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: pgmap v99: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nvmeof 2024-04-30T09:59:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: Upgrade: Finalizing container_image settings 2024-04-30T09:59:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: Upgrade: Complete! 2024-04-30T09:59:01.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: pgmap v100: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:01.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: pgmap v101: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:59:01.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: pgmap v102: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:59:01.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: pgmap v103: 161 pgs: 37 peering, 124 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail 2024-04-30T09:59:01.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:01.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:01.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:01.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:01 smithi046 ceph-mon[65689]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 30/627 objects degraded (4.785%), 6 pgs degraded) 2024-04-30T09:59:03.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:03 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:03 smithi026 ceph-mon[80969]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-04-30T09:59:03.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:03 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:03.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:03 smithi026 ceph-mon[86310]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-04-30T09:59:03.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:03 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:03.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:03 smithi046 ceph-mon[65689]: pgmap v104: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 2 op/s 2024-04-30T09:59:05.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:05 smithi026 ceph-mon[86310]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:05.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:05 smithi026 ceph-mon[80969]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:05 smithi046 ceph-mon[65689]: pgmap v105: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:59:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:59:06] "GET /metrics HTTP/1.1" 200 38168 "" "Prometheus/2.43.0" 2024-04-30T09:59:07.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:07 smithi026 ceph-mon[86310]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-04-30T09:59:07.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:07 smithi026 ceph-mon[80969]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-04-30T09:59:07.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:07 smithi046 ceph-mon[65689]: pgmap v106: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-04-30T09:59:09.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:09 smithi026 ceph-mon[86310]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:09.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:09 smithi026 ceph-mon[80969]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:09.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:09 smithi046 ceph-mon[65689]: pgmap v107: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:11.841 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:11 smithi026 ceph-mon[86310]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:59:11.841 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:11 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:11.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:11 smithi026 ceph-mon[80969]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:59:11.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:11 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:11.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:11 smithi046 ceph-mon[65689]: pgmap v108: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:59:11.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:11 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:13.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:13 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:13.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:13 smithi026 ceph-mon[86310]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:13 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:13 smithi026 ceph-mon[80969]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:13.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:13 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:13.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:13 smithi046 ceph-mon[65689]: pgmap v109: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:15.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:15 smithi026 ceph-mon[86310]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:15.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:15 smithi026 ceph-mon[80969]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:15.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:15 smithi046 ceph-mon[65689]: pgmap v110: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:59:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:59:16] "GET /metrics HTTP/1.1" 200 38164 "" "Prometheus/2.43.0" 2024-04-30T09:59:17.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:17.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:17.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:16 smithi026 ceph-mon[86310]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:17.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:17.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:17.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:16 smithi026 ceph-mon[80969]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:16 smithi046 ceph-mon[65689]: pgmap v111: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:19 smithi046 ceph-mon[65689]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:19.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:19 smithi026 ceph-mon[86310]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:19 smithi026 ceph-mon[80969]: pgmap v112: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:21.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:21 smithi046 ceph-mon[65689]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:21.852 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:21 smithi026 ceph-mon[86310]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:21.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:21 smithi026 ceph-mon[80969]: pgmap v113: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:23.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:23 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:23.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:23 smithi026 ceph-mon[86310]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:23.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:23 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:23.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:23 smithi026 ceph-mon[80969]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:23.623 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T09:59:23.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:23 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:23.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:23 smithi046 ceph-mon[65689]: pgmap v114: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:24.626 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:24 smithi026 ceph-mon[86310]: from='client.54253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:24.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:24 smithi026 ceph-mon[80969]: from='client.54253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:24.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:24 smithi046 ceph-mon[65689]: from='client.54253 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:25.682 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (5m) 30s ago 27m 25.7M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (5m) 2m ago 27m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 30s ago 26m 62.1M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (6m) 2m ago 32m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (16m) 30s ago 34m 505M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (3m) 30s ago 34m 48.1M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (3m) 2m ago 32m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (2m) 30s ago 32m 40.0M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) 30s ago 28m 15.8M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (6m) 2m ago 28m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (55s) 30s ago 32m 45.8M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T09:59:25.683 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (35s) 30s ago 31m 12.7M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T09:59:25.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (31m) 30s ago 31m 54.3M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:59:25.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (30m) 30s ago 30m 54.0M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:59:25.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (30m) 2m ago 30m 58.3M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:59:25.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (30m) 2m ago 30m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:59:25.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (29m) 2m ago 29m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:59:25.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (29m) 2m ago 29m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:59:25.684 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (5m) 2m ago 28m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:59:25.684 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (27m) 30s ago 27m 77.4M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:59:25.684 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (27m) 2m ago 27m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:59:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:25 smithi026 ceph-mon[86310]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:25 smithi026 ceph-mon[80969]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:25.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:25 smithi046 ceph-mon[65689]: pgmap v115: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:26.362 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-30T09:59:26.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:59:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:59:26] "GET /metrics HTTP/1.1" 200 38164 "" "Prometheus/2.43.0" 2024-04-30T09:59:27.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:27 smithi026 ceph-mon[80969]: from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:27.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:27 smithi026 ceph-mon[80969]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:27.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:27 smithi026 ceph-mon[86310]: from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:27.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:27 smithi026 ceph-mon[86310]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:27.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:27 smithi046 ceph-mon[65689]: from='client.44319 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:27.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:27 smithi046 ceph-mon[65689]: pgmap v116: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:28.456 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:59:28.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:28 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/115829533' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:28.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:28 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/115829533' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:28 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/115829533' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:29.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 7'"'"'' 2024-04-30T09:59:29.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:29 smithi026 ceph-mon[86310]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:29.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:29 smithi026 ceph-mon[80969]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:29.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:29 smithi046 ceph-mon[65689]: pgmap v117: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:31.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:31 smithi026 ceph-mon[86310]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:31.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:31.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:31 smithi026 ceph-mon[80969]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:31.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:31 smithi046 ceph-mon[65689]: pgmap v118: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:31.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:32.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:32 smithi026 ceph-mon[86310]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:32.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:32 smithi026 ceph-mon[80969]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:32.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:32 smithi046 ceph-mon[65689]: from='client.54271 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:33.282 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:59:33.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:33 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:33.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:33 smithi026 ceph-mon[86310]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:33 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:33.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:33 smithi026 ceph-mon[80969]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:33.920 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade status' 2024-04-30T09:59:33.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:33 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:33.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:33 smithi046 ceph-mon[65689]: pgmap v119: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:35.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:35 smithi026 ceph-mon[80969]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:35.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:35 smithi026 ceph-mon[86310]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:35.967 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:59:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-30T09:59:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-30T09:59:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-30T09:59:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:59:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-30T09:59:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-30T09:59:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:59:35.968 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:59:35.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:35 smithi046 ceph-mon[65689]: pgmap v120: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:36.612 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T09:59:36.635 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:59:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:59:36] "GET /metrics HTTP/1.1" 200 38168 "" "Prometheus/2.43.0" 2024-04-30T09:59:37.894 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:37 smithi026 ceph-mon[86310]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:37.894 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:37 smithi026 ceph-mon[86310]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:37.894 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:37 smithi026 ceph-mon[80969]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:37.894 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:37 smithi026 ceph-mon[80969]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:37.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:37 smithi046 ceph-mon[65689]: from='client.54277 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:37.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:37 smithi046 ceph-mon[65689]: pgmap v121: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:38.620 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:59:38.620 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T09:59:38.620 INFO:teuthology.orchestra.run.smithi026.stdout: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T09:59:39.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1' 2024-04-30T09:59:39.830 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:39 smithi026 ceph-mon[86310]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:39.830 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:39 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/678476079' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:59:39.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:39 smithi026 ceph-mon[80969]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:39.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:39 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/678476079' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:59:39.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:39 smithi046 ceph-mon[65689]: pgmap v122: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:39.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:39 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/678476079' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T09:59:41.870 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:41 smithi026 ceph-mon[80969]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:41.870 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:41 smithi026 ceph-mon[86310]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:41.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:41 smithi046 ceph-mon[65689]: pgmap v123: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T09:59:42.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:42 smithi026 ceph-mon[86310]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:42.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:42 smithi026 ceph-mon[80969]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:42.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:42 smithi046 ceph-mon[65689]: from='client.54289 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "crash,osd", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:43.266 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:59:43.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:43 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:43.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:43 smithi026 ceph-mon[86310]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:43.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:43.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:43.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:43 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:43.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:43 smithi026 ceph-mon[80969]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:43.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:43.878 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:43.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-30T09:59:43.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:43 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:43.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:43 smithi046 ceph-mon[65689]: pgmap v124: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T09:59:43.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:43.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:44.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:44 smithi026 ceph-mon[86310]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:59:44.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:44 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:44.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:44 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:44.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:44 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:44.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:44 smithi026 ceph-mon[80969]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:59:44.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:44 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:44.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:44 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:44.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:44 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:44 smithi046 ceph-mon[65689]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:59:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:44 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:44 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T09:59:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:44 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:45.533 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:45 smithi026 ceph-mon[80969]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 893 B/s rd, 0 op/s 2024-04-30T09:59:45.885 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:45 smithi046 ceph-mon[65689]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 893 B/s rd, 0 op/s 2024-04-30T09:59:45.885 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:45 smithi046 ceph-mon[65689]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:45.885 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:45 smithi046 ceph-mon[65689]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:59:45.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:45 smithi026 ceph-mon[80969]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:45.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:45 smithi026 ceph-mon[80969]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:59:45.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:45 smithi026 ceph-mon[86310]: pgmap v125: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 893 B/s rd, 0 op/s 2024-04-30T09:59:45.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:45 smithi026 ceph-mon[86310]: pgmap v126: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:45.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:45 smithi026 ceph-mon[86310]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T09:59:45.967 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T09:59:46.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:46.664 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:46.665 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:59:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:59:46] "GET /metrics HTTP/1.1" 200 38165 "" "Prometheus/2.43.0" 2024-04-30T09:59:46.665 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T09:59:46.665 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (5m) 51s ago 28m 25.7M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T09:59:46.665 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (5m) 3m ago 28m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T09:59:46.665 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 51s ago 27m 62.1M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T09:59:46.665 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (7m) 3m ago 32m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T09:59:46.665 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (17m) 51s ago 34m 505M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T09:59:46.665 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (3m) 51s ago 34m 48.1M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (4m) 3m ago 33m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (3m) 51s ago 33m 40.0M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) 51s ago 28m 15.8M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (6m) 3m ago 28m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (76s) 51s ago 32m 45.8M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (56s) 51s ago 32m 12.7M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (31m) 51s ago 31m 54.3M 2993M 17.2.0 e1d6a67b021e d6b6f2a088b0 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (31m) 51s ago 31m 54.0M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (30m) 3m ago 30m 58.3M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (30m) 3m ago 30m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (30m) 3m ago 30m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (29m) 3m ago 29m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (6m) 3m ago 29m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T09:59:46.666 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (27m) 51s ago 27m 77.4M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T09:59:46.667 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (27m) 3m ago 27m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T09:59:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T09:59:47.093 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:59:47.093 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T09:59:47.093 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T09:59:47.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:59:47.093 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T09:59:47.093 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:59:47.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:59:47.093 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 6, 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 8, 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 7 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T09:59:47.094 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:59:47.449 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T09:59:47.449 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", 2024-04-30T09:59:47.449 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T09:59:47.450 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd. Upgrade limited to 1 daemons (1 remaining).", 2024-04-30T09:59:47.450 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T09:59:47.450 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "", 2024-04-30T09:59:47.450 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 image", 2024-04-30T09:59:47.450 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T09:59:47.450 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T09:59:47.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:47 smithi026 ceph-mon[80969]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:47.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:47 smithi026 ceph-mon[80969]: from='client.44355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:47.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:47 smithi026 ceph-mon[80969]: from='client.44361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:47.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:47 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2261949560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:47 smithi026 ceph-mon[86310]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:47 smithi026 ceph-mon[86310]: from='client.44355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:47 smithi026 ceph-mon[86310]: from='client.44361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:47 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2261949560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:47.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:47 smithi046 ceph-mon[65689]: pgmap v127: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:47.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:47 smithi046 ceph-mon[65689]: from='client.44355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:47.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:47 smithi046 ceph-mon[65689]: from='client.44361 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:47.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:47 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2261949560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:48.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:48.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:48.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:48 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-30T09:59:48.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='client.54307 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:48.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='client.54319 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T09:59:48.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:48.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T09:59:48.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:48.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:48 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-30T09:59:49.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:59:49.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:59:49.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mgr 2024-04-30T09:59:49.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mon 2024-04-30T09:59:49.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all crash 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: Upgrade: osd.2 is safe to restart 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mgr 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mon 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all crash 2024-04-30T09:59:49.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-30T09:59:49.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: Upgrade: osd.2 is safe to restart 2024-04-30T09:59:49.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:49.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:49.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-30T09:59:49.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:49 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:49.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T09:59:49.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T09:59:49.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mgr 2024-04-30T09:59:49.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mon 2024-04-30T09:59:49.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all crash 2024-04-30T09:59:49.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-30T09:59:49.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: Upgrade: osd.2 is safe to restart 2024-04-30T09:59:49.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: pgmap v128: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:49.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:49.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-30T09:59:49.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:49 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T09:59:50.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:50 smithi026 ceph-mon[86310]: Upgrade: Updating osd.2 2024-04-30T09:59:50.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:50 smithi026 ceph-mon[86310]: Deploying daemon osd.2 on smithi026 2024-04-30T09:59:50.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:50 smithi026 ceph-mon[80969]: Upgrade: Updating osd.2 2024-04-30T09:59:50.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:50 smithi026 ceph-mon[80969]: Deploying daemon osd.2 on smithi026 2024-04-30T09:59:50.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:50 smithi046 ceph-mon[65689]: Upgrade: Updating osd.2 2024-04-30T09:59:50.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:50 smithi046 ceph-mon[65689]: Deploying daemon osd.2 on smithi026 2024-04-30T09:59:51.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:51 smithi026 ceph-mon[86310]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:51.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:51 smithi026 ceph-mon[80969]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:51.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:51 smithi046 ceph-mon[65689]: pgmap v129: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:52.778 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:52 smithi026 systemd[1]: Stopping Ceph osd.2 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:59:53.198 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[35911]: 2024-04-30T09:59:52.776+0000 7fa651851700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T09:59:53.198 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[35911]: 2024-04-30T09:59:52.776+0000 7fa651851700 -1 osd.2 123 *** Got signal Terminated *** 2024-04-30T09:59:53.198 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[35911]: 2024-04-30T09:59:52.776+0000 7fa651851700 -1 osd.2 123 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T09:59:53.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:53 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:53.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:53 smithi026 ceph-mon[86310]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:53.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:53 smithi026 ceph-mon[86310]: osd.2 marked itself down and dead 2024-04-30T09:59:53.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:53 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:53.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:53 smithi026 ceph-mon[80969]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:53.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:53 smithi026 ceph-mon[80969]: osd.2 marked itself down and dead 2024-04-30T09:59:53.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:53 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T09:59:53.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:53 smithi046 ceph-mon[65689]: pgmap v130: 161 pgs: 161 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T09:59:53.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:53 smithi046 ceph-mon[65689]: osd.2 marked itself down and dead 2024-04-30T09:59:54.179 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:53 smithi026 podman[104680]: 2024-04-30 09:59:53.860796783 +0000 UTC m=+1.188824154 container died d6b6f2a088b0200c060b26ff12b3290468885af6ca6890970f668fee3c779d7e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2, RELEASE=HEAD, version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., vcs-type=git, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, ceph=True, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870) 2024-04-30T09:59:54.446 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:54 smithi026 podman[104680]: 2024-04-30 09:59:54.251161345 +0000 UTC m=+1.579188720 container cleanup d6b6f2a088b0200c060b26ff12b3290468885af6ca6890970f668fee3c779d7e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, version=8, architecture=x86_64, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, GIT_CLEAN=True, io.buildah.version=1.19.8, RELEASE=HEAD, ceph=True, name=centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-type=git, GIT_BRANCH=HEAD, io.openshift.expose-services=, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream) 2024-04-30T09:59:54.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:54 smithi026 bash[104680]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2 2024-04-30T09:59:54.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:54 smithi026 ceph-mon[86310]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T09:59:54.814 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:54 smithi026 ceph-mon[86310]: osdmap e124: 8 total, 7 up, 8 in 2024-04-30T09:59:54.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:54 smithi026 ceph-mon[80969]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T09:59:54.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:54 smithi026 ceph-mon[80969]: osdmap e124: 8 total, 7 up, 8 in 2024-04-30T09:59:54.814 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:54 smithi026 podman[104698]: 2024-04-30 09:59:54.485750054 +0000 UTC m=+0.614142602 container remove d6b6f2a088b0200c060b26ff12b3290468885af6ca6890970f668fee3c779d7e (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , io.openshift.tags=base centos centos-stream, distribution-scope=public, version=8, name=centos-stream, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, release=754, io.buildah.version=1.19.8, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.component=centos-stream-container, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_CLEAN=True) 2024-04-30T09:59:54.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:54 smithi046 ceph-mon[65689]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T09:59:54.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:54 smithi046 ceph-mon[65689]: osdmap e124: 8 total, 7 up, 8 in 2024-04-30T09:59:55.090 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:54 smithi026 podman[104789]: 2024-04-30 09:59:54.7164262 +0000 UTC m=+0.018763941 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:59:55.090 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:54 smithi026 podman[104789]: 2024-04-30 09:59:54.828370825 +0000 UTC m=+0.130712949 container create b00d6488affa1d5295d52cb9023cd1cee63b4feb186f06efdaadfd4488195acb (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T09:59:55.090 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:55 smithi026 podman[104789]: 2024-04-30 09:59:55.006553395 +0000 UTC m=+0.308891140 container init b00d6488affa1d5295d52cb9023cd1cee63b4feb186f06efdaadfd4488195acb (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-30T09:59:55.090 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:55 smithi026 podman[104789]: 2024-04-30 09:59:55.010172007 +0000 UTC m=+0.312509732 container start b00d6488affa1d5295d52cb9023cd1cee63b4feb186f06efdaadfd4488195acb (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-30T09:59:55.091 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:55 smithi026 podman[104789]: 2024-04-30 09:59:55.089305131 +0000 UTC m=+0.391642864 container attach b00d6488affa1d5295d52cb9023cd1cee63b4feb186f06efdaadfd4488195acb (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-30T09:59:55.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:55 smithi026 podman[104789]: 2024-04-30 09:59:55.312676879 +0000 UTC m=+0.615014606 container died b00d6488affa1d5295d52cb9023cd1cee63b4feb186f06efdaadfd4488195acb (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T09:59:55.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:55 smithi026 ceph-mon[86310]: pgmap v132: 161 pgs: 7 peering, 11 stale+active+clean, 143 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:59:55.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:55 smithi026 ceph-mon[86310]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-30T09:59:55.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:55 smithi026 ceph-mon[86310]: osdmap e125: 8 total, 7 up, 8 in 2024-04-30T09:59:55.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:55 smithi026 ceph-mon[80969]: pgmap v132: 161 pgs: 7 peering, 11 stale+active+clean, 143 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:59:55.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:55 smithi026 ceph-mon[80969]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-30T09:59:55.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:55 smithi026 ceph-mon[80969]: osdmap e125: 8 total, 7 up, 8 in 2024-04-30T09:59:55.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:55 smithi046 ceph-mon[65689]: pgmap v132: 161 pgs: 7 peering, 11 stale+active+clean, 143 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T09:59:55.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:55 smithi046 ceph-mon[65689]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-30T09:59:55.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:55 smithi046 ceph-mon[65689]: osdmap e125: 8 total, 7 up, 8 in 2024-04-30T09:59:56.196 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:55 smithi026 podman[104815]: 2024-04-30 09:59:55.926406961 +0000 UTC m=+0.607876241 container remove b00d6488affa1d5295d52cb9023cd1cee63b4feb186f06efdaadfd4488195acb (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-30T09:59:56.197 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:55 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-30T09:59:56.197 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:55 smithi026 systemd[1]: Stopped Ceph osd.2 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:59:56.197 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:55 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.2.service: Consumed 13.887s CPU time. 2024-04-30T09:59:56.197 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:56 smithi026 systemd[1]: Starting Ceph osd.2 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T09:59:56.664 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:56 smithi026 ceph-mon[86310]: pgmap v134: 161 pgs: 2 active+undersized+degraded, 7 peering, 9 stale+active+clean, 5 active+undersized, 138 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 19/627 objects degraded (3.030%) 2024-04-30T09:59:56.664 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:56 smithi026 ceph-mon[86310]: Health check failed: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded (PG_DEGRADED) 2024-04-30T09:59:56.664 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 09:59:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:09:59:56] "GET /metrics HTTP/1.1" 200 38165 "" "Prometheus/2.43.0" 2024-04-30T09:59:56.665 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:56 smithi026 podman[104921]: 2024-04-30 09:59:56.290702789 +0000 UTC m=+0.019102598 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:59:56.665 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:56 smithi026 podman[104921]: 2024-04-30 09:59:56.394565316 +0000 UTC m=+0.122965180 container create c4b9885a860d8a11e2553f22734261af2407685f3e2528aa5d0f427b3887aaad (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T09:59:56.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:56 smithi026 ceph-mon[80969]: pgmap v134: 161 pgs: 2 active+undersized+degraded, 7 peering, 9 stale+active+clean, 5 active+undersized, 138 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 19/627 objects degraded (3.030%) 2024-04-30T09:59:56.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:56 smithi026 ceph-mon[80969]: Health check failed: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded (PG_DEGRADED) 2024-04-30T09:59:56.947 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:56 smithi026 podman[104921]: 2024-04-30 09:59:56.664064524 +0000 UTC m=+0.392464333 container init c4b9885a860d8a11e2553f22734261af2407685f3e2528aa5d0f427b3887aaad (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-30T09:59:56.947 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:56 smithi026 podman[104921]: 2024-04-30 09:59:56.667551954 +0000 UTC m=+0.395951773 container start c4b9885a860d8a11e2553f22734261af2407685f3e2528aa5d0f427b3887aaad (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-04-30T09:59:56.947 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:56 smithi026 podman[104921]: 2024-04-30 09:59:56.772655655 +0000 UTC m=+0.501055474 container attach c4b9885a860d8a11e2553f22734261af2407685f3e2528aa5d0f427b3887aaad (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T09:59:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:56 smithi046 ceph-mon[65689]: pgmap v134: 161 pgs: 2 active+undersized+degraded, 7 peering, 9 stale+active+clean, 5 active+undersized, 138 active+clean; 585 KiB data, 132 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 19/627 objects degraded (3.030%) 2024-04-30T09:59:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:56 smithi046 ceph-mon[65689]: Health check failed: Degraded data redundancy: 19/627 objects degraded (3.030%), 2 pgs degraded (PG_DEGRADED) 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate[104936]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate[104936]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 bash[104921]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 bash[104921]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 bash[104921]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 bash[104921]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 bash[104921]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 bash[104921]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 bash[104921]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-30T09:59:57.447 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 bash[104921]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-30T09:59:57.448 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 bash[104921]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-30T09:59:57.448 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate[104936]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-30T09:59:57.448 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate[104936]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-30T09:59:57.448 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate[104936]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-30T09:59:57.448 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate[104936]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-30T09:59:57.448 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate[104936]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-30T09:59:57.448 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate[104936]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-30T09:59:57.448 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate[104936]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-30T09:59:57.448 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 podman[104921]: 2024-04-30 09:59:57.187789609 +0000 UTC m=+0.916189419 container died c4b9885a860d8a11e2553f22734261af2407685f3e2528aa5d0f427b3887aaad (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T09:59:57.946 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 podman[105061]: 2024-04-30 09:59:57.726351665 +0000 UTC m=+0.527851727 container remove c4b9885a860d8a11e2553f22734261af2407685f3e2528aa5d0f427b3887aaad (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-30T09:59:58.272 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 podman[105128]: 2024-04-30 09:59:57.851176019 +0000 UTC m=+0.018767469 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T09:59:58.272 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:57 smithi026 podman[105128]: 2024-04-30 09:59:57.952171884 +0000 UTC m=+0.119763321 container create 170943dc4c2ad3990fc520cce24dd16d4cc324de7da95df463c7d33cc8aed065 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True) 2024-04-30T09:59:58.272 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:58 smithi026 podman[105128]: 2024-04-30 09:59:58.171999923 +0000 UTC m=+0.339591369 container init 170943dc4c2ad3990fc520cce24dd16d4cc324de7da95df463c7d33cc8aed065 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T09:59:58.272 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:58 smithi026 podman[105128]: 2024-04-30 09:59:58.175546993 +0000 UTC m=+0.343138431 container start 170943dc4c2ad3990fc520cce24dd16d4cc324de7da95df463c7d33cc8aed065 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True) 2024-04-30T09:59:58.272 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:58 smithi026 ceph-osd[105148]: -- 172.21.15.26:0/1743024215 <== mon.1 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e335945680 con 0x55e33590f000 2024-04-30T09:59:58.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:58 smithi026 bash[105128]: 170943dc4c2ad3990fc520cce24dd16d4cc324de7da95df463c7d33cc8aed065 2024-04-30T09:59:58.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:58 smithi026 systemd[1]: Started Ceph osd.2 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T09:59:59.178 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 09:59:58 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[105144]: 2024-04-30T09:59:58.717+0000 7f3dc50ce7c0 -1 Falling back to public interface 2024-04-30T09:59:59.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[80969]: pgmap v135: 161 pgs: 12 active+undersized+degraded, 7 peering, 28 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 50/627 objects degraded (7.974%) 2024-04-30T09:59:59.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:59.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:59.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:59.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[86310]: pgmap v135: 161 pgs: 12 active+undersized+degraded, 7 peering, 28 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 50/627 objects degraded (7.974%) 2024-04-30T09:59:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 09:59:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T09:59:59.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:59 smithi046 ceph-mon[65689]: pgmap v135: 161 pgs: 12 active+undersized+degraded, 7 peering, 28 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 50/627 objects degraded (7.974%) 2024-04-30T09:59:59.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:59.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T09:59:59.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 09:59:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state']; 1 osds down; Reduced data availability: 2 pgs inactive, 7 pgs peering; Degraded data redundancy: 50/627 objects degraded (7.974%), 12 pgs degraded 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: [WRN] OSD_DOWN: 1 osds down 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: osd.2 (root=default,host=smithi026) is down 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: [WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs inactive, 7 pgs peering 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 3.1f is stuck peering for 61s, current state peering, last acting [0,5] 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 4.6 is stuck peering for 61s, current state peering, last acting [0,1] 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 5.8 is stuck peering for 63s, current state peering, last acting [0,1] 2024-04-30T10:00:00.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 5.b is stuck peering for 83s, current state peering, last acting [0,5] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 5.1e is stuck peering for 61s, current state peering, last acting [0,7] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 6.0 is stuck peering for 61s, current state peering, last acting [0,3] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 6.9 is stuck peering for 61s, current state peering, last acting [0,7] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: [WRN] PG_DEGRADED: Degraded data redundancy: 50/627 objects degraded (7.974%), 12 pgs degraded 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 4.d is active+undersized+degraded, acting [4,1] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 4.1f is active+undersized+degraded, acting [6,5] 2024-04-30T10:00:00.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[80969]: pg 5.1c is active+undersized+degraded, acting [4,3] 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state']; 1 osds down; Reduced data availability: 2 pgs inactive, 7 pgs peering; Degraded data redundancy: 50/627 objects degraded (7.974%), 12 pgs degraded 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: [WRN] OSD_DOWN: 1 osds down 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: osd.2 (root=default,host=smithi026) is down 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: [WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs inactive, 7 pgs peering 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 3.1f is stuck peering for 61s, current state peering, last acting [0,5] 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 4.6 is stuck peering for 61s, current state peering, last acting [0,1] 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 5.8 is stuck peering for 63s, current state peering, last acting [0,1] 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 5.b is stuck peering for 83s, current state peering, last acting [0,5] 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 5.1e is stuck peering for 61s, current state peering, last acting [0,7] 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 6.0 is stuck peering for 61s, current state peering, last acting [0,3] 2024-04-30T10:00:00.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 6.9 is stuck peering for 61s, current state peering, last acting [0,7] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: [WRN] PG_DEGRADED: Degraded data redundancy: 50/627 objects degraded (7.974%), 12 pgs degraded 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 4.d is active+undersized+degraded, acting [4,1] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 4.1f is active+undersized+degraded, acting [6,5] 2024-04-30T10:00:00.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:00 smithi026 ceph-mon[86310]: pg 5.1c is active+undersized+degraded, acting [4,3] 2024-04-30T10:00:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state']; 1 osds down; Reduced data availability: 2 pgs inactive, 7 pgs peering; Degraded data redundancy: 50/627 objects degraded (7.974%), 12 pgs degraded 2024-04-30T10:00:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:00:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T10:00:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: [WRN] OSD_DOWN: 1 osds down 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: osd.2 (root=default,host=smithi026) is down 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: [WRN] PG_AVAILABILITY: Reduced data availability: 2 pgs inactive, 7 pgs peering 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 3.1f is stuck peering for 61s, current state peering, last acting [0,5] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 4.6 is stuck peering for 61s, current state peering, last acting [0,1] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 5.8 is stuck peering for 63s, current state peering, last acting [0,1] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 5.b is stuck peering for 83s, current state peering, last acting [0,5] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 5.1e is stuck peering for 61s, current state peering, last acting [0,7] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 6.0 is stuck peering for 61s, current state peering, last acting [0,3] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 6.9 is stuck peering for 61s, current state peering, last acting [0,7] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: [WRN] PG_DEGRADED: Degraded data redundancy: 50/627 objects degraded (7.974%), 12 pgs degraded 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 2.3 is active+undersized+degraded, acting [5,7] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 2.7 is active+undersized+degraded, acting [6,7] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 3.0 is active+undersized+degraded, acting [1,6] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 3.4 is active+undersized+degraded, acting [1,5] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 3.5 is active+undersized+degraded, acting [5,3] 2024-04-30T10:00:00.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 3.9 is active+undersized+degraded, acting [4,7] 2024-04-30T10:00:00.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 3.13 is active+undersized+degraded, acting [7,4] 2024-04-30T10:00:00.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 3.1a is active+undersized+degraded, acting [4,1] 2024-04-30T10:00:00.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 3.1e is active+undersized+degraded, acting [3,6] 2024-04-30T10:00:00.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 4.d is active+undersized+degraded, acting [4,1] 2024-04-30T10:00:00.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 4.1f is active+undersized+degraded, acting [6,5] 2024-04-30T10:00:00.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:00 smithi046 ceph-mon[65689]: pg 5.1c is active+undersized+degraded, acting [4,3] 2024-04-30T10:00:01.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-mon[86310]: pgmap v136: 161 pgs: 13 active+undersized+degraded, 34 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 55/627 objects degraded (8.772%) 2024-04-30T10:00:01.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-mon[86310]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 7 pgs peering) 2024-04-30T10:00:01.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:01.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:01.621 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[105144]: 2024-04-30T10:00:01.444+0000 7f3dc50ce7c0 -1 osd.2 0 read_superblock omap replica is missing. 2024-04-30T10:00:01.621 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[105144]: 2024-04-30T10:00:01.466+0000 7f3dc50ce7c0 -1 osd.2 123 log_to_monitors true 2024-04-30T10:00:01.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-mon[80969]: pgmap v136: 161 pgs: 13 active+undersized+degraded, 34 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 55/627 objects degraded (8.772%) 2024-04-30T10:00:01.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-mon[80969]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 7 pgs peering) 2024-04-30T10:00:01.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:01.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:01 smithi046 ceph-mon[65689]: pgmap v136: 161 pgs: 13 active+undersized+degraded, 34 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 0 op/s; 55/627 objects degraded (8.772%) 2024-04-30T10:00:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:01 smithi046 ceph-mon[65689]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 7 pgs peering) 2024-04-30T10:00:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:02.697 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:00:02 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[105144]: 2024-04-30T10:00:02.360+0000 7f3db6525700 -1 osd.2 123 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T10:00:02.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:02 smithi026 ceph-mon[80969]: from='osd.2 [v2:172.21.15.26:6818/123714870,v1:172.21.15.26:6819/123714870]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-30T10:00:02.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:02 smithi026 ceph-mon[80969]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-30T10:00:02.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:02 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 55/627 objects degraded (8.772%), 13 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:02.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:02 smithi026 ceph-mon[86310]: from='osd.2 [v2:172.21.15.26:6818/123714870,v1:172.21.15.26:6819/123714870]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-30T10:00:02.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:02 smithi026 ceph-mon[86310]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-30T10:00:02.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:02 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 55/627 objects degraded (8.772%), 13 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:02.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:02 smithi046 ceph-mon[65689]: from='osd.2 [v2:172.21.15.26:6818/123714870,v1:172.21.15.26:6819/123714870]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-30T10:00:02.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:02 smithi046 ceph-mon[65689]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-30T10:00:02.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:02 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 55/627 objects degraded (8.772%), 13 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:03.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:03.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[86310]: pgmap v137: 161 pgs: 13 active+undersized+degraded, 34 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 55/627 objects degraded (8.772%) 2024-04-30T10:00:03.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[86310]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-30T10:00:03.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[86310]: from='osd.2 [v2:172.21.15.26:6818/123714870,v1:172.21.15.26:6819/123714870]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:03.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[86310]: osdmap e126: 8 total, 7 up, 8 in 2024-04-30T10:00:03.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[86310]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:03.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:03.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:03.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:03.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[80969]: pgmap v137: 161 pgs: 13 active+undersized+degraded, 34 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 55/627 objects degraded (8.772%) 2024-04-30T10:00:03.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[80969]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-30T10:00:03.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[80969]: from='osd.2 [v2:172.21.15.26:6818/123714870,v1:172.21.15.26:6819/123714870]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:03.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[80969]: osdmap e126: 8 total, 7 up, 8 in 2024-04-30T10:00:03.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[80969]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:03.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:03.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:03.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:03 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:03.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:03 smithi046 ceph-mon[65689]: pgmap v137: 161 pgs: 13 active+undersized+degraded, 34 active+undersized, 114 active+clean; 585 KiB data, 133 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 55/627 objects degraded (8.772%) 2024-04-30T10:00:03.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:03 smithi046 ceph-mon[65689]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-30T10:00:03.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:03 smithi046 ceph-mon[65689]: from='osd.2 [v2:172.21.15.26:6818/123714870,v1:172.21.15.26:6819/123714870]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:03.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:03 smithi046 ceph-mon[65689]: osdmap e126: 8 total, 7 up, 8 in 2024-04-30T10:00:03.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:03 smithi046 ceph-mon[65689]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:03.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:03.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:04.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[86310]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:00:04.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[86310]: osd.2 [v2:172.21.15.26:6818/123714870,v1:172.21.15.26:6819/123714870] boot 2024-04-30T10:00:04.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[86310]: osdmap e127: 8 total, 8 up, 8 in 2024-04-30T10:00:04.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T10:00:04.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:04.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:04.669 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[86310]: osdmap e128: 8 total, 8 up, 8 in 2024-04-30T10:00:04.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[80969]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:00:04.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[80969]: osd.2 [v2:172.21.15.26:6818/123714870,v1:172.21.15.26:6819/123714870] boot 2024-04-30T10:00:04.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[80969]: osdmap e127: 8 total, 8 up, 8 in 2024-04-30T10:00:04.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T10:00:04.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:04.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:04.670 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:04 smithi026 ceph-mon[80969]: osdmap e128: 8 total, 8 up, 8 in 2024-04-30T10:00:04.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:04 smithi046 ceph-mon[65689]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:00:04.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:04 smithi046 ceph-mon[65689]: osd.2 [v2:172.21.15.26:6818/123714870,v1:172.21.15.26:6819/123714870] boot 2024-04-30T10:00:04.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:04 smithi046 ceph-mon[65689]: osdmap e127: 8 total, 8 up, 8 in 2024-04-30T10:00:04.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-30T10:00:04.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:04.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:04.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:04 smithi046 ceph-mon[65689]: osdmap e128: 8 total, 8 up, 8 in 2024-04-30T10:00:05.684 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:05 smithi026 ceph-mon[86310]: pgmap v140: 161 pgs: 5 peering, 12 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 50/627 objects degraded (7.974%) 2024-04-30T10:00:05.684 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:05 smithi026 ceph-mon[80969]: pgmap v140: 161 pgs: 5 peering, 12 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 50/627 objects degraded (7.974%) 2024-04-30T10:00:05.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:05 smithi046 ceph-mon[65689]: pgmap v140: 161 pgs: 5 peering, 12 active+undersized+degraded, 30 active+undersized, 114 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s; 50/627 objects degraded (7.974%) 2024-04-30T10:00:06.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:00:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:00:06] "GET /metrics HTTP/1.1" 200 38170 "" "Prometheus/2.43.0" 2024-04-30T10:00:07.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:07 smithi026 ceph-mon[86310]: pgmap v142: 161 pgs: 5 peering, 10 active+undersized+degraded, 27 active+undersized, 119 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 31/627 objects degraded (4.944%) 2024-04-30T10:00:07.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:07 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:07.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:07 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 31/627 objects degraded (4.944%), 10 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:07.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:07 smithi026 ceph-mon[80969]: pgmap v142: 161 pgs: 5 peering, 10 active+undersized+degraded, 27 active+undersized, 119 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 31/627 objects degraded (4.944%) 2024-04-30T10:00:07.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:07 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:07.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:07 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 31/627 objects degraded (4.944%), 10 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:07 smithi046 ceph-mon[65689]: pgmap v142: 161 pgs: 5 peering, 10 active+undersized+degraded, 27 active+undersized, 119 active+clean; 585 KiB data, 148 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 31/627 objects degraded (4.944%) 2024-04-30T10:00:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:07 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:07 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 31/627 objects degraded (4.944%), 10 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:09.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: pgmap v143: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: pgmap v144: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all nvmeof 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: Upgrade: Finalizing container_image settings 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T10:00:09.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: pgmap v143: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: pgmap v144: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all nvmeof 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: Upgrade: Finalizing container_image settings 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T10:00:09.201 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: Upgrade: Complete! 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: pgmap v145: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: pgmap v146: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:09.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: Upgrade: Complete! 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: pgmap v145: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: pgmap v146: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:09.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:09.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: pgmap v147: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T10:00:09.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: pgmap v148: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-30T10:00:09.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:09.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:09.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: pgmap v147: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T10:00:09.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: pgmap v148: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-30T10:00:09.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:08 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: pgmap v143: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 872 B/s rd, 0 op/s 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: pgmap v144: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 942 B/s rd, 0 op/s 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nvmeof 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: Upgrade: Finalizing container_image settings 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T10:00:09.236 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: Upgrade: Complete! 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T10:00:09.237 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: pgmap v145: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: pgmap v146: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: pgmap v147: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: pgmap v148: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 2 op/s 2024-04-30T10:00:09.238 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:08 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:10.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:09 smithi026 ceph-mon[86310]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 10 pgs degraded) 2024-04-30T10:00:10.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:09 smithi026 ceph-mon[80969]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 10 pgs degraded) 2024-04-30T10:00:10.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:09 smithi046 ceph-mon[65689]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 31/627 objects degraded (4.944%), 10 pgs degraded) 2024-04-30T10:00:11.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:10 smithi026 ceph-mon[86310]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:00:11.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:10 smithi026 ceph-mon[80969]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:00:11.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:10 smithi046 ceph-mon[65689]: pgmap v149: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:00:12.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:12 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:12.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:12 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:12.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:12 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:13 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:13 smithi026 ceph-mon[86310]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:13 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:13.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:13 smithi026 ceph-mon[80969]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:13 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:13 smithi046 ceph-mon[65689]: pgmap v150: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:15.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:15 smithi026 ceph-mon[86310]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-04-30T10:00:15.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:15 smithi026 ceph-mon[80969]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-04-30T10:00:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:15 smithi046 ceph-mon[65689]: pgmap v151: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 833 B/s rd, 0 op/s 2024-04-30T10:00:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:00:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:00:16] "GET /metrics HTTP/1.1" 200 38170 "" "Prometheus/2.43.0" 2024-04-30T10:00:17.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:17.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:17.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:16 smithi026 ceph-mon[86310]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-30T10:00:17.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:17.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:17.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:17.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:16 smithi026 ceph-mon[80969]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-30T10:00:17.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:16 smithi046 ceph-mon[65689]: pgmap v152: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 628 B/s rd, 0 op/s 2024-04-30T10:00:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:18.504 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T10:00:19.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:19 smithi026 ceph-mon[86310]: from='client.44391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:19 smithi026 ceph-mon[86310]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:00:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:19 smithi026 ceph-mon[80969]: from='client.44391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:19 smithi026 ceph-mon[80969]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:00:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:19 smithi046 ceph-mon[65689]: from='client.44391 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:19 smithi046 ceph-mon[65689]: pgmap v153: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:00:20.507 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (6m) 17s ago 28m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (6m) 3m ago 28m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 17s ago 27m 62.2M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (7m) 3m ago 33m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (17m) 17s ago 35m 507M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (3m) 17s ago 35m 51.1M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (4m) 3m ago 33m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (3m) 17s ago 33m 42.2M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (7m) 17s ago 29m 15.9M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (7m) 3m ago 29m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (110s) 17s ago 33m 46.7M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (90s) 17s ago 32m 43.8M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (22s) 17s ago 32m 12.7M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:00:20.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (31m) 17s ago 31m 55.5M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T10:00:20.509 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (31m) 3m ago 31m 58.3M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T10:00:20.509 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (31m) 3m ago 31m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T10:00:20.509 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (30m) 3m ago 30m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T10:00:20.509 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (30m) 3m ago 30m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T10:00:20.509 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (6m) 3m ago 29m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:00:20.509 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (28m) 17s ago 28m 77.7M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T10:00:20.509 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (27m) 3m ago 27m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T10:00:21.077 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 2'"'"'' 2024-04-30T10:00:21.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:21 smithi026 ceph-mon[86310]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:21.400 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:21 smithi026 ceph-mon[86310]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:21.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:21 smithi026 ceph-mon[80969]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:21.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:21 smithi026 ceph-mon[80969]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:21 smithi046 ceph-mon[65689]: pgmap v154: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:21.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:21 smithi046 ceph-mon[65689]: from='client.54337 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:23.191 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:00:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:23 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:23 smithi026 ceph-mon[86310]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:23 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:23 smithi026 ceph-mon[80969]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:23 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:23 smithi046 ceph-mon[65689]: pgmap v155: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:23.824 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '"'"'.up_to_date | length == 8'"'"'' 2024-04-30T10:00:24.445 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:24 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2332712311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:24.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:24 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2332712311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:24 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2332712311' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:25.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:25 smithi026 ceph-mon[80969]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:25.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:25 smithi026 ceph-mon[86310]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:25.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:25 smithi046 ceph-mon[65689]: pgmap v156: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:00:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:00:26] "GET /metrics HTTP/1.1" 200 38170 "" "Prometheus/2.43.0" 2024-04-30T10:00:27.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:27 smithi026 ceph-mon[86310]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:27.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:27 smithi026 ceph-mon[86310]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:27.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:27 smithi026 ceph-mon[80969]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:27.205 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:27 smithi026 ceph-mon[80969]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:27 smithi046 ceph-mon[65689]: from='client.54349 -' entity='client.admin' cmd=[{"prefix": "orch upgrade check", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:27.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:27 smithi046 ceph-mon[65689]: pgmap v157: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:28.258 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:00:28.889 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade status' 2024-04-30T10:00:29.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:29 smithi026 ceph-mon[86310]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:29.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:29 smithi026 ceph-mon[80969]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:29 smithi046 ceph-mon[65689]: pgmap v158: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:30.954 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:00:30.955 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-30T10:00:30.955 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-30T10:00:30.955 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-30T10:00:30.955 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T10:00:30.955 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-30T10:00:30.955 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-30T10:00:30.955 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T10:00:30.955 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:00:31.315 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:31 smithi026 ceph-mon[86310]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:31.315 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:31.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:31 smithi026 ceph-mon[80969]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:31.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:31 smithi046 ceph-mon[65689]: pgmap v159: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:31.643 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T10:00:32.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:32 smithi026 ceph-mon[80969]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:32.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:32 smithi026 ceph-mon[86310]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:32.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:32 smithi046 ceph-mon[65689]: from='client.54355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:33.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:33 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:33.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:33 smithi026 ceph-mon[80969]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:33.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:33 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:33.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:33 smithi026 ceph-mon[86310]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:33 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:33.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:33 smithi046 ceph-mon[65689]: pgmap v160: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:33.920 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:00:33.920 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:00:33.920 INFO:teuthology.orchestra.run.smithi026.stdout: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T10:00:34.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:34 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2101459332' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:00:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:34 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2101459332' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:00:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:34 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2101459332' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:00:34.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd' 2024-04-30T10:00:35.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:35 smithi026 ceph-mon[86310]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:35.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:35 smithi026 ceph-mon[80969]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:35.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:35 smithi046 ceph-mon[65689]: pgmap v161: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:00:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:00:36] "GET /metrics HTTP/1.1" 200 38175 "" "Prometheus/2.43.0" 2024-04-30T10:00:37.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:37 smithi026 ceph-mon[86310]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:37.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:37 smithi026 ceph-mon[80969]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:37.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:37 smithi046 ceph-mon[65689]: pgmap v162: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:00:38.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:38 smithi026 ceph-mon[86310]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:38 smithi026 ceph-mon[80969]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:38 smithi046 ceph-mon[65689]: from='client.54367 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "daemon_types": "crash,osd", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:38.756 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:00:39.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[86310]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:39.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:39.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:39.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:39.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:39.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:39.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[80969]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:39.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:39.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:39.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:39.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:39.351 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:39.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-30T10:00:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:39 smithi046 ceph-mon[65689]: pgmap v163: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:00:39.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:40.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:40 smithi026 ceph-mon[86310]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:00:40.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:40 smithi026 ceph-mon[86310]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-04-30T10:00:40.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:40 smithi026 ceph-mon[86310]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:40.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:40 smithi026 ceph-mon[86310]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:00:40.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:40 smithi026 ceph-mon[80969]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:00:40.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:40 smithi026 ceph-mon[80969]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-04-30T10:00:40.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:40 smithi026 ceph-mon[80969]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:40.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:40 smithi026 ceph-mon[80969]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:00:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:40 smithi046 ceph-mon[65689]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:00:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:40 smithi046 ceph-mon[65689]: pgmap v164: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 959 B/s rd, 0 op/s 2024-04-30T10:00:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:40 smithi046 ceph-mon[65689]: pgmap v165: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:40 smithi046 ceph-mon[65689]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:00:41.602 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:00:42.297 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (6m) 39s ago 29m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (6m) 4m ago 28m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 39s ago 28m 62.2M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (8m) 4m ago 33m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (18m) 39s ago 35m 507M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (4m) 39s ago 35m 51.1M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (5m) 4m ago 34m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (3m) 39s ago 34m 42.2M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (7m) 39s ago 29m 15.9M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (7m) 4m ago 29m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (2m) 39s ago 33m 46.7M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (112s) 39s ago 33m 43.8M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:00:42.298 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (44s) 39s ago 32m 12.7M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:00:42.299 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (32m) 39s ago 32m 55.5M 2993M 17.2.0 e1d6a67b021e e8d13dac0bc4 2024-04-30T10:00:42.299 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (31m) 4m ago 31m 58.3M 3249M 17.2.0 e1d6a67b021e 93023274f043 2024-04-30T10:00:42.299 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (31m) 4m ago 31m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T10:00:42.299 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (31m) 4m ago 31m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T10:00:42.299 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (30m) 4m ago 30m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T10:00:42.299 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (7m) 4m ago 29m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:00:42.299 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (28m) 39s ago 28m 77.7M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T10:00:42.299 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (28m) 4m ago 28m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T10:00:42.435 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:42 smithi026 ceph-mon[80969]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-04-30T10:00:42.435 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:42 smithi026 ceph-mon[86310]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-04-30T10:00:42.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:42 smithi046 ceph-mon[65689]: pgmap v166: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 590 B/s rd, 0 op/s 2024-04-30T10:00:42.723 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:00:42.723 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T10:00:42.723 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T10:00:42.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 7, 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 8 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T10:00:42.724 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:00:43.087 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:00:43.087 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T10:00:43.087 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T10:00:43.087 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-30T10:00:43.087 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T10:00:43.087 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "3/8 daemons upgraded", 2024-04-30T10:00:43.087 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-30T10:00:43.087 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T10:00:43.087 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:00:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='client.54373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='client.44439 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='client.44445 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T10:00:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T10:00:43.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mgr 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mon 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all crash 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: Upgrade: osd.3 is safe to restart 2024-04-30T10:00:43.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/4235587087' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='client.54373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='client.44439 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='client.44445 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mgr 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mon 2024-04-30T10:00:43.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all crash 2024-04-30T10:00:43.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-30T10:00:43.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-30T10:00:43.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: Upgrade: osd.3 is safe to restart 2024-04-30T10:00:43.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:43 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/4235587087' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='client.54373 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:43.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='client.44439 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='client.44445 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mgr 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mon 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all crash 2024-04-30T10:00:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:43.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-30T10:00:43.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-30T10:00:43.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: Upgrade: osd.3 is safe to restart 2024-04-30T10:00:43.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:43 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/4235587087' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:00:44.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[86310]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:44.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[86310]: from='client.44457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:44.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:44.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-30T10:00:44.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:44.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[80969]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:44.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[80969]: from='client.44457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:44.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:44.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-30T10:00:44.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:44 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:44 smithi046 ceph-mon[65689]: pgmap v167: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:44 smithi046 ceph-mon[65689]: from='client.44457 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:00:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:44 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:44 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-30T10:00:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:44 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:00:45.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:45 smithi026 ceph-mon[86310]: Upgrade: Updating osd.3 2024-04-30T10:00:45.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:45 smithi026 ceph-mon[86310]: Deploying daemon osd.3 on smithi026 2024-04-30T10:00:45.769 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:45 smithi026 ceph-mon[80969]: Upgrade: Updating osd.3 2024-04-30T10:00:45.769 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:45 smithi026 ceph-mon[80969]: Deploying daemon osd.3 on smithi026 2024-04-30T10:00:45.885 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:45 smithi046 ceph-mon[65689]: Upgrade: Updating osd.3 2024-04-30T10:00:45.885 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:45 smithi046 ceph-mon[65689]: Deploying daemon osd.3 on smithi026 2024-04-30T10:00:46.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:00:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:00:46] "GET /metrics HTTP/1.1" 200 38176 "" "Prometheus/2.43.0" 2024-04-30T10:00:46.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:46 smithi026 ceph-mon[86310]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:46.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:46.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:46 smithi026 ceph-mon[80969]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:46.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:46 smithi046 ceph-mon[65689]: pgmap v168: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:00:47.705 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:47 smithi026 systemd[1]: Stopping Ceph osd.3 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:00:47.980 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:47 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[38920]: 2024-04-30T10:00:47.704+0000 7f27590ba700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:00:47.980 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:47 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[38920]: 2024-04-30T10:00:47.704+0000 7f27590ba700 -1 osd.3 128 *** Got signal Terminated *** 2024-04-30T10:00:47.980 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:47 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[38920]: 2024-04-30T10:00:47.704+0000 7f27590ba700 -1 osd.3 128 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:00:48.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:48 smithi026 ceph-mon[86310]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:48.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:48 smithi026 ceph-mon[86310]: osd.3 marked itself down and dead 2024-04-30T10:00:48.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:48 smithi026 ceph-mon[80969]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:48.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:48 smithi026 ceph-mon[80969]: osd.3 marked itself down and dead 2024-04-30T10:00:48.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:48 smithi046 ceph-mon[65689]: pgmap v169: 161 pgs: 161 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:00:48.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:48 smithi046 ceph-mon[65689]: osd.3 marked itself down and dead 2024-04-30T10:00:49.306 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:48 smithi026 podman[110985]: 2024-04-30 10:00:48.990856081 +0000 UTC m=+1.387954385 container died e8d13dac0bc4d7f86f82ab160aa0efb5289374c66fc1ea27c1233f567a391ab1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3, architecture=x86_64, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, CEPH_POINT_RELEASE=-17.2.0, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, version=8, io.openshift.tags=base centos centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, RELEASE=HEAD, vcs-type=git, maintainer=Guillaume Abrioux , GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, name=centos-stream, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, release=754, io.buildah.version=1.19.8, io.openshift.expose-services=) 2024-04-30T10:00:49.657 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:49 smithi026 ceph-mon[86310]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:00:49.657 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:49 smithi026 ceph-mon[86310]: osdmap e129: 8 total, 7 up, 8 in 2024-04-30T10:00:49.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:49 smithi026 ceph-mon[80969]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:00:49.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:49 smithi026 ceph-mon[80969]: osdmap e129: 8 total, 7 up, 8 in 2024-04-30T10:00:49.657 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:49 smithi026 podman[110985]: 2024-04-30 10:00:49.396921811 +0000 UTC m=+1.794020114 container cleanup e8d13dac0bc4d7f86f82ab160aa0efb5289374c66fc1ea27c1233f567a391ab1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.openshift.tags=base centos centos-stream, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, io.k8s.display-name=CentOS Stream 8, vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, vcs-type=git, maintainer=Guillaume Abrioux , ceph=True, name=centos-stream, com.redhat.component=centos-stream-container) 2024-04-30T10:00:49.658 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:49 smithi026 bash[110985]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3 2024-04-30T10:00:49.947 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:49 smithi026 podman[111002]: 2024-04-30 10:00:49.656630143 +0000 UTC m=+0.657236740 container remove e8d13dac0bc4d7f86f82ab160aa0efb5289374c66fc1ea27c1233f567a391ab1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, architecture=x86_64, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, maintainer=Guillaume Abrioux , RELEASE=HEAD, com.redhat.component=centos-stream-container, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, vcs-type=git, release=754) 2024-04-30T10:00:49.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:49 smithi046 ceph-mon[65689]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:00:49.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:49 smithi046 ceph-mon[65689]: osdmap e129: 8 total, 7 up, 8 in 2024-04-30T10:00:50.235 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:49 smithi026 podman[111092]: 2024-04-30 10:00:49.876132844 +0000 UTC m=+0.018356858 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:00:50.235 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:49 smithi026 podman[111092]: 2024-04-30 10:00:49.97404498 +0000 UTC m=+0.116268967 container create 902ad4c405b15a05606507bcfacece91674c7a06a8c7e70cd7670e78172c203c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424) 2024-04-30T10:00:50.235 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:50 smithi026 podman[111092]: 2024-04-30 10:00:50.159968859 +0000 UTC m=+0.302192849 container init 902ad4c405b15a05606507bcfacece91674c7a06a8c7e70cd7670e78172c203c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True) 2024-04-30T10:00:50.235 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:50 smithi026 podman[111092]: 2024-04-30 10:00:50.163133786 +0000 UTC m=+0.305357782 container start 902ad4c405b15a05606507bcfacece91674c7a06a8c7e70cd7670e78172c203c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:00:50.235 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:50 smithi026 podman[111092]: 2024-04-30 10:00:50.234871053 +0000 UTC m=+0.377095043 container attach 902ad4c405b15a05606507bcfacece91674c7a06a8c7e70cd7670e78172c203c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:00:50.526 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:50 smithi026 podman[111092]: 2024-04-30 10:00:50.463747769 +0000 UTC m=+0.605971771 container died 902ad4c405b15a05606507bcfacece91674c7a06a8c7e70cd7670e78172c203c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424) 2024-04-30T10:00:50.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:50 smithi026 ceph-mon[80969]: pgmap v171: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:00:50.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:50 smithi026 ceph-mon[80969]: osdmap e130: 8 total, 7 up, 8 in 2024-04-30T10:00:50.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:50 smithi026 ceph-mon[86310]: pgmap v171: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:00:50.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:50 smithi026 ceph-mon[86310]: osdmap e130: 8 total, 7 up, 8 in 2024-04-30T10:00:50.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:50 smithi046 ceph-mon[65689]: pgmap v171: 161 pgs: 27 stale+active+clean, 134 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:00:50.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:50 smithi046 ceph-mon[65689]: osdmap e130: 8 total, 7 up, 8 in 2024-04-30T10:00:51.447 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 podman[111118]: 2024-04-30 10:00:51.038985658 +0000 UTC m=+0.569442663 container remove 902ad4c405b15a05606507bcfacece91674c7a06a8c7e70cd7670e78172c203c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-30T10:00:51.447 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-30T10:00:51.447 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 systemd[1]: Stopped Ceph osd.3 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:00:51.447 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.3.service: Consumed 18.277s CPU time. 2024-04-30T10:00:51.447 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 systemd[1]: Starting Ceph osd.3 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:00:51.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:51 smithi026 ceph-mon[80969]: pgmap v173: 161 pgs: 5 active+undersized+degraded, 14 peering, 16 stale+active+clean, 8 active+undersized, 118 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 18/627 objects degraded (2.871%) 2024-04-30T10:00:51.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:51 smithi026 ceph-mon[80969]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-04-30T10:00:51.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:51 smithi026 ceph-mon[80969]: Health check failed: Degraded data redundancy: 18/627 objects degraded (2.871%), 5 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:51.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:51 smithi026 ceph-mon[86310]: pgmap v173: 161 pgs: 5 active+undersized+degraded, 14 peering, 16 stale+active+clean, 8 active+undersized, 118 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 18/627 objects degraded (2.871%) 2024-04-30T10:00:51.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:51 smithi026 ceph-mon[86310]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-04-30T10:00:51.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:51 smithi026 ceph-mon[86310]: Health check failed: Degraded data redundancy: 18/627 objects degraded (2.871%), 5 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:51.819 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 podman[111223]: 2024-04-30 10:00:51.473531494 +0000 UTC m=+0.106391435 container create a5ddec66038105dd3a5c06bd315c1d8d71fe466ead7d96fe0767249ded98d69c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-30T10:00:51.819 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 podman[111223]: 2024-04-30 10:00:51.385875039 +0000 UTC m=+0.018734999 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:00:51.819 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 podman[111223]: 2024-04-30 10:00:51.709396187 +0000 UTC m=+0.342256128 container init a5ddec66038105dd3a5c06bd315c1d8d71fe466ead7d96fe0767249ded98d69c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424) 2024-04-30T10:00:51.819 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 podman[111223]: 2024-04-30 10:00:51.712750821 +0000 UTC m=+0.345610762 container start a5ddec66038105dd3a5c06bd315c1d8d71fe466ead7d96fe0767249ded98d69c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5) 2024-04-30T10:00:51.819 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:51 smithi026 podman[111223]: 2024-04-30 10:00:51.817841202 +0000 UTC m=+0.450701158 container attach a5ddec66038105dd3a5c06bd315c1d8d71fe466ead7d96fe0767249ded98d69c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0) 2024-04-30T10:00:51.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:51 smithi046 ceph-mon[65689]: pgmap v173: 161 pgs: 5 active+undersized+degraded, 14 peering, 16 stale+active+clean, 8 active+undersized, 118 active+clean; 585 KiB data, 149 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 0 op/s; 18/627 objects degraded (2.871%) 2024-04-30T10:00:51.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:51 smithi046 ceph-mon[65689]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-04-30T10:00:51.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:51 smithi046 ceph-mon[65689]: Health check failed: Degraded data redundancy: 18/627 objects degraded (2.871%), 5 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:52.581 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate[111238]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate[111238]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 bash[111223]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 bash[111223]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 bash[111223]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 bash[111223]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 bash[111223]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 bash[111223]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 bash[111223]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 bash[111223]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 bash[111223]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate[111238]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate[111238]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate[111238]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate[111238]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-30T10:00:52.582 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate[111238]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-30T10:00:52.583 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate[111238]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-30T10:00:52.583 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate[111238]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-30T10:00:52.583 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 podman[111223]: 2024-04-30 10:00:52.230526548 +0000 UTC m=+0.863386489 container died a5ddec66038105dd3a5c06bd315c1d8d71fe466ead7d96fe0767249ded98d69c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-30T10:00:52.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:52.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:52 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:52.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:52 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:00:53.195 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:52 smithi026 podman[111362]: 2024-04-30 10:00:52.880628039 +0000 UTC m=+0.644826965 container remove a5ddec66038105dd3a5c06bd315c1d8d71fe466ead7d96fe0767249ded98d69c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-04-30T10:00:53.195 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:53 smithi026 podman[111423]: 2024-04-30 10:00:53.081425446 +0000 UTC m=+0.093520315 container create 8d72616b503d7ca10188afe6c9148b810187e1d84c2f3481884dedbe357ce35c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-30T10:00:53.195 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:53 smithi026 podman[111423]: 2024-04-30 10:00:53.00694448 +0000 UTC m=+0.019039360 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:00:53.697 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:53 smithi026 podman[111423]: 2024-04-30 10:00:53.308961864 +0000 UTC m=+0.321056747 container init 8d72616b503d7ca10188afe6c9148b810187e1d84c2f3481884dedbe357ce35c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-30T10:00:53.697 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:53 smithi026 podman[111423]: 2024-04-30 10:00:53.312566887 +0000 UTC m=+0.324661757 container start 8d72616b503d7ca10188afe6c9148b810187e1d84c2f3481884dedbe357ce35c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:00:53.697 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:53 smithi026 ceph-osd[111441]: -- 172.21.15.26:0/938481537 <== mon.1 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a22ea0b680 con 0x55a22e9d4c00 2024-04-30T10:00:53.697 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:53 smithi026 bash[111423]: 8d72616b503d7ca10188afe6c9148b810187e1d84c2f3481884dedbe357ce35c 2024-04-30T10:00:53.697 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:53 smithi026 systemd[1]: Started Ceph osd.3 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:00:54.196 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:53 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[111437]: 2024-04-30T10:00:53.857+0000 7f8ed08e47c0 -1 Falling back to public interface 2024-04-30T10:00:54.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:54 smithi026 ceph-mon[86310]: pgmap v174: 161 pgs: 18 active+undersized+degraded, 14 peering, 34 active+undersized, 95 active+clean; 585 KiB data, 150 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-04-30T10:00:54.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:54 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:54.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:54 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:54.525 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:54 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:54.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:54 smithi026 ceph-mon[80969]: pgmap v174: 161 pgs: 18 active+undersized+degraded, 14 peering, 34 active+undersized, 95 active+clean; 585 KiB data, 150 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-04-30T10:00:54.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:54 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:54.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:54 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:54.526 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:54 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:54.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:54 smithi046 ceph-mon[65689]: pgmap v174: 161 pgs: 18 active+undersized+degraded, 14 peering, 34 active+undersized, 95 active+clean; 585 KiB data, 150 MiB used, 715 GiB / 715 GiB avail; 57/627 objects degraded (9.091%) 2024-04-30T10:00:54.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:54 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:54.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:54 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:54.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:54 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:00:55.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:55 smithi026 ceph-mon[86310]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-04-30T10:00:55.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:55 smithi026 ceph-mon[80969]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-04-30T10:00:55.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:55 smithi046 ceph-mon[65689]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2024-04-30T10:00:56.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-mon[86310]: pgmap v175: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 150 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-04-30T10:00:56.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-mon[86310]: from='osd.3 [v2:172.21.15.26:6826/2970763770,v1:172.21.15.26:6827/2970763770]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-30T10:00:56.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-mon[86310]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-30T10:00:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-mon[80969]: pgmap v175: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 150 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-04-30T10:00:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-mon[80969]: from='osd.3 [v2:172.21.15.26:6826/2970763770,v1:172.21.15.26:6827/2970763770]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-30T10:00:56.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-mon[80969]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-30T10:00:56.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:00:56] "GET /metrics HTTP/1.1" 200 38176 "" "Prometheus/2.43.0" 2024-04-30T10:00:56.698 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[111437]: 2024-04-30T10:00:56.333+0000 7f8ed08e47c0 -1 osd.3 0 read_superblock omap replica is missing. 2024-04-30T10:00:56.698 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[111437]: 2024-04-30T10:00:56.366+0000 7f8ed08e47c0 -1 osd.3 128 log_to_monitors true 2024-04-30T10:00:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:56 smithi046 ceph-mon[65689]: pgmap v175: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 150 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-04-30T10:00:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:56 smithi046 ceph-mon[65689]: from='osd.3 [v2:172.21.15.26:6826/2970763770,v1:172.21.15.26:6827/2970763770]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-30T10:00:56.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:56 smithi046 ceph-mon[65689]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-30T10:00:57.196 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:00:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[111437]: 2024-04-30T10:00:56.795+0000 7f8ebdd33700 -1 osd.3 128 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T10:00:57.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[86310]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-30T10:00:57.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[86310]: osdmap e131: 8 total, 7 up, 8 in 2024-04-30T10:00:57.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[86310]: from='osd.3 [v2:172.21.15.26:6826/2970763770,v1:172.21.15.26:6827/2970763770]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:57.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[86310]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:57.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[86310]: from='osd.3 ' entity='osd.3' 2024-04-30T10:00:57.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 79/627 objects degraded (12.600%), 23 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:57.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[80969]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-30T10:00:57.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[80969]: osdmap e131: 8 total, 7 up, 8 in 2024-04-30T10:00:57.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[80969]: from='osd.3 [v2:172.21.15.26:6826/2970763770,v1:172.21.15.26:6827/2970763770]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:57.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[80969]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:57.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[80969]: from='osd.3 ' entity='osd.3' 2024-04-30T10:00:57.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:57 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 79/627 objects degraded (12.600%), 23 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:57.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:57 smithi046 ceph-mon[65689]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-30T10:00:57.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:57 smithi046 ceph-mon[65689]: osdmap e131: 8 total, 7 up, 8 in 2024-04-30T10:00:57.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:57 smithi046 ceph-mon[65689]: from='osd.3 [v2:172.21.15.26:6826/2970763770,v1:172.21.15.26:6827/2970763770]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:57.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:57 smithi046 ceph-mon[65689]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2024-04-30T10:00:57.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:57 smithi046 ceph-mon[65689]: from='osd.3 ' entity='osd.3' 2024-04-30T10:00:57.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:57 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 79/627 objects degraded (12.600%), 23 pgs degraded (PG_DEGRADED) 2024-04-30T10:00:58.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[86310]: pgmap v177: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 165 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-04-30T10:00:58.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[86310]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:00:58.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[86310]: osd.3 [v2:172.21.15.26:6826/2970763770,v1:172.21.15.26:6827/2970763770] boot 2024-04-30T10:00:58.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[86310]: osdmap e132: 8 total, 8 up, 8 in 2024-04-30T10:00:58.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T10:00:58.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:58.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[80969]: pgmap v177: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 165 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-04-30T10:00:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[80969]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:00:58.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[80969]: osd.3 [v2:172.21.15.26:6826/2970763770,v1:172.21.15.26:6827/2970763770] boot 2024-04-30T10:00:58.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[80969]: osdmap e132: 8 total, 8 up, 8 in 2024-04-30T10:00:58.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T10:00:58.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:58.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:58.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:58 smithi046 ceph-mon[65689]: pgmap v177: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 165 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-04-30T10:00:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:58 smithi046 ceph-mon[65689]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:00:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:58 smithi046 ceph-mon[65689]: osd.3 [v2:172.21.15.26:6826/2970763770,v1:172.21.15.26:6827/2970763770] boot 2024-04-30T10:00:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:58 smithi046 ceph-mon[65689]: osdmap e132: 8 total, 8 up, 8 in 2024-04-30T10:00:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-30T10:00:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:00:58.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:00.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:59 smithi026 ceph-mon[86310]: pgmap v179: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 165 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-04-30T10:01:00.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:59 smithi026 ceph-mon[86310]: osdmap e133: 8 total, 8 up, 8 in 2024-04-30T10:01:00.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:00.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:00:59 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:00.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:59 smithi026 ceph-mon[80969]: pgmap v179: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 165 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-04-30T10:01:00.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:59 smithi026 ceph-mon[80969]: osdmap e133: 8 total, 8 up, 8 in 2024-04-30T10:01:00.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:00.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:00:59 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:59 smithi046 ceph-mon[65689]: pgmap v179: 161 pgs: 23 active+undersized+degraded, 43 active+undersized, 95 active+clean; 585 KiB data, 165 MiB used, 715 GiB / 715 GiB avail; 79/627 objects degraded (12.600%) 2024-04-30T10:01:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:59 smithi046 ceph-mon[65689]: osdmap e133: 8 total, 8 up, 8 in 2024-04-30T10:01:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:00.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:00:59 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:02.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:01 smithi026 ceph-mon[86310]: pgmap v181: 161 pgs: 17 active+undersized+degraded, 33 active+undersized, 111 active+clean; 585 KiB data, 165 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 55/627 objects degraded (8.772%) 2024-04-30T10:01:02.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:02.127 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:02.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:01 smithi026 ceph-mon[80969]: pgmap v181: 161 pgs: 17 active+undersized+degraded, 33 active+undersized, 111 active+clean; 585 KiB data, 165 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 55/627 objects degraded (8.772%) 2024-04-30T10:01:02.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:02.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:01 smithi046 ceph-mon[65689]: pgmap v181: 161 pgs: 17 active+undersized+degraded, 33 active+undersized, 111 active+clean; 585 KiB data, 165 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 55/627 objects degraded (8.772%) 2024-04-30T10:01:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:02.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:03.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:02 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:03.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:02 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 55/627 objects degraded (8.772%), 17 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:03.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:02 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:03.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:02 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 55/627 objects degraded (8.772%), 17 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:03.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:02 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:03.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:02 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 55/627 objects degraded (8.772%), 17 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:03.907 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:03.907 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/627 objects degraded (8.772%), 17 pgs degraded) 2024-04-30T10:01:03.907 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-30T10:01:03.908 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:03 smithi046 ceph-mon[65689]: Upgrade: osd.4 is safe to restart 2024-04-30T10:01:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/627 objects degraded (8.772%), 17 pgs degraded) 2024-04-30T10:01:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:04.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-30T10:01:04.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[86310]: Upgrade: osd.4 is safe to restart 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: pgmap v182: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 55/627 objects degraded (8.772%), 17 pgs degraded) 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: pgmap v183: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: pgmap v184: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:04.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:04.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:04.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:04.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-30T10:01:04.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-30T10:01:04.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:03 smithi026 ceph-mon[80969]: Upgrade: osd.4 is safe to restart 2024-04-30T10:01:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:05 smithi046 ceph-mon[65689]: Upgrade: Updating osd.4 2024-04-30T10:01:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-30T10:01:05.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:05.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:05 smithi046 ceph-mon[65689]: Deploying daemon osd.4 on smithi046 2024-04-30T10:01:05.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:05 smithi046 ceph-mon[65689]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:06.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[86310]: Upgrade: Updating osd.4 2024-04-30T10:01:06.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[86310]: Deploying daemon osd.4 on smithi046 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[86310]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[80969]: Upgrade: Updating osd.4 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[80969]: Deploying daemon osd.4 on smithi046 2024-04-30T10:01:06.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:05 smithi026 ceph-mon[80969]: pgmap v185: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:01:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:01:06] "GET /metrics HTTP/1.1" 200 38036 "" "Prometheus/2.43.0" 2024-04-30T10:01:08.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:08 smithi026 ceph-mon[86310]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:08.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:08 smithi026 ceph-mon[80969]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:08.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:08 smithi046 ceph-mon[65689]: pgmap v186: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:08.484 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:08 smithi046 systemd[1]: Stopping Ceph osd.4 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:01:08.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[26611]: 2024-04-30T10:01:08.590+0000 7ff8096e8700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:01:08.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[26611]: 2024-04-30T10:01:08.590+0000 7ff8096e8700 -1 osd.4 133 *** Got signal Terminated *** 2024-04-30T10:01:08.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[26611]: 2024-04-30T10:01:08.590+0000 7ff8096e8700 -1 osd.4 133 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:01:09.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:09 smithi026 ceph-mon[86310]: osd.4 marked itself down and dead 2024-04-30T10:01:09.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:09 smithi026 ceph-mon[80969]: osd.4 marked itself down and dead 2024-04-30T10:01:09.471 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:09 smithi046 ceph-mon[65689]: osd.4 marked itself down and dead 2024-04-30T10:01:09.734 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:09 smithi046 podman[75981]: 2024-04-30 10:01:09.470509352 +0000 UTC m=+0.999021854 container died 93023274f0430d11db80db1c349dc9f1127e5035f3a104383cc88722cc4a77f1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4, ceph=True, io.buildah.version=1.19.8, architecture=x86_64, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, com.redhat.component=centos-stream-container, RELEASE=HEAD, name=centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_BRANCH=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, vcs-type=git, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_CLEAN=True) 2024-04-30T10:01:10.084 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:09 smithi046 podman[75981]: 2024-04-30 10:01:09.867515847 +0000 UTC m=+1.396028349 container cleanup 93023274f0430d11db80db1c349dc9f1127e5035f3a104383cc88722cc4a77f1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=754, io.openshift.expose-services=, GIT_CLEAN=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, maintainer=Guillaume Abrioux , com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, io.buildah.version=1.19.8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, name=centos-stream, ceph=True, com.redhat.component=centos-stream-container, architecture=x86_64, distribution-scope=public, io.k8s.display-name=CentOS Stream 8, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git) 2024-04-30T10:01:10.084 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:09 smithi046 bash[75981]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4 2024-04-30T10:01:10.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:10 smithi026 ceph-mon[80969]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:10.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:10 smithi026 ceph-mon[80969]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:01:10.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:10 smithi026 ceph-mon[80969]: osdmap e134: 8 total, 7 up, 8 in 2024-04-30T10:01:10.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:10 smithi026 ceph-mon[86310]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:10.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:10 smithi026 ceph-mon[86310]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:01:10.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:10 smithi026 ceph-mon[86310]: osdmap e134: 8 total, 7 up, 8 in 2024-04-30T10:01:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:10 smithi046 ceph-mon[65689]: pgmap v187: 161 pgs: 161 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:01:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:10 smithi046 ceph-mon[65689]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:01:10.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:10 smithi046 ceph-mon[65689]: osdmap e134: 8 total, 7 up, 8 in 2024-04-30T10:01:10.484 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:10 smithi046 podman[75999]: 2024-04-30 10:01:10.083474914 +0000 UTC m=+0.614215670 container remove 93023274f0430d11db80db1c349dc9f1127e5035f3a104383cc88722cc4a77f1 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, release=754, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, version=8, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, name=centos-stream, io.buildah.version=1.19.8, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, ceph=True) 2024-04-30T10:01:10.779 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:10 smithi046 podman[76093]: 2024-04-30 10:01:10.414040198 +0000 UTC m=+0.018589205 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:01:10.779 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:10 smithi046 podman[76093]: 2024-04-30 10:01:10.517800808 +0000 UTC m=+0.122349813 container create 82c5d0815c774cf10ecc1fa1918287cad3d136ce92075e0ae70a7ea5d24b661a (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=) 2024-04-30T10:01:10.779 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:10 smithi046 podman[76093]: 2024-04-30 10:01:10.704020647 +0000 UTC m=+0.308569654 container init 82c5d0815c774cf10ecc1fa1918287cad3d136ce92075e0ae70a7ea5d24b661a (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5) 2024-04-30T10:01:10.779 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:10 smithi046 podman[76093]: 2024-04-30 10:01:10.707401654 +0000 UTC m=+0.311950660 container start 82c5d0815c774cf10ecc1fa1918287cad3d136ce92075e0ae70a7ea5d24b661a (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-04-30T10:01:10.779 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:10 smithi046 podman[76093]: 2024-04-30 10:01:10.778655884 +0000 UTC m=+0.383204905 container attach 82c5d0815c774cf10ecc1fa1918287cad3d136ce92075e0ae70a7ea5d24b661a (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-30T10:01:11.104 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:11 smithi046 podman[76093]: 2024-04-30 10:01:11.012921868 +0000 UTC m=+0.617470866 container died 82c5d0815c774cf10ecc1fa1918287cad3d136ce92075e0ae70a7ea5d24b661a (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:01:11.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:11 smithi026 ceph-mon[80969]: osdmap e135: 8 total, 7 up, 8 in 2024-04-30T10:01:11.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:11 smithi026 ceph-mon[86310]: osdmap e135: 8 total, 7 up, 8 in 2024-04-30T10:01:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:11 smithi046 ceph-mon[65689]: osdmap e135: 8 total, 7 up, 8 in 2024-04-30T10:01:11.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:11 smithi046 podman[76118]: 2024-04-30 10:01:11.567395316 +0000 UTC m=+0.548965508 container remove 82c5d0815c774cf10ecc1fa1918287cad3d136ce92075e0ae70a7ea5d24b661a (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:01:11.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:11 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-30T10:01:11.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:11 smithi046 systemd[1]: Stopped Ceph osd.4 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:01:11.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:11 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.4.service: Consumed 18.438s CPU time. 2024-04-30T10:01:11.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:11 smithi046 systemd[1]: Starting Ceph osd.4 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:01:12.305 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-mon[65689]: pgmap v190: 161 pgs: 11 peering, 20 stale+active+clean, 130 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-30T10:01:12.305 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-mon[65689]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-30T10:01:12.305 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:12.305 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 podman[76224]: 2024-04-30 10:01:11.923118713 +0000 UTC m=+0.019390339 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:01:12.306 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 podman[76224]: 2024-04-30 10:01:12.035154182 +0000 UTC m=+0.131425796 container create 6c6ff9e469ec996336638d396d6f67d304b7eefb2a47352927164bdcb218835b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-30T10:01:12.306 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 podman[76224]: 2024-04-30 10:01:12.230522153 +0000 UTC m=+0.326793778 container init 6c6ff9e469ec996336638d396d6f67d304b7eefb2a47352927164bdcb218835b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD) 2024-04-30T10:01:12.306 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 podman[76224]: 2024-04-30 10:01:12.233969275 +0000 UTC m=+0.330240890 container start 6c6ff9e469ec996336638d396d6f67d304b7eefb2a47352927164bdcb218835b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-04-30T10:01:12.306 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 podman[76224]: 2024-04-30 10:01:12.304792932 +0000 UTC m=+0.401064556 container attach 6c6ff9e469ec996336638d396d6f67d304b7eefb2a47352927164bdcb218835b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:01:12.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:12 smithi026 ceph-mon[80969]: pgmap v190: 161 pgs: 11 peering, 20 stale+active+clean, 130 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-30T10:01:12.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:12 smithi026 ceph-mon[80969]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-30T10:01:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:12 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:12.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:12 smithi026 ceph-mon[86310]: pgmap v190: 161 pgs: 11 peering, 20 stale+active+clean, 130 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-30T10:01:12.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:12 smithi026 ceph-mon[86310]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-30T10:01:12.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:12 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:12.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate[76240]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:01:12.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate[76240]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:01:12.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 bash[76224]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:01:12.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 bash[76224]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:01:12.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 bash[76224]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 bash[76224]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 bash[76224]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 bash[76224]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 bash[76224]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 bash[76224]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 bash[76224]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate[76240]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate[76240]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate[76240]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate[76240]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate[76240]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate[76240]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate[76240]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-30T10:01:12.985 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:12 smithi046 podman[76224]: 2024-04-30 10:01:12.753206339 +0000 UTC m=+0.849477956 container died 6c6ff9e469ec996336638d396d6f67d304b7eefb2a47352927164bdcb218835b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:01:13.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:13 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:13.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:13 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:13.466 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:01:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:13 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:13.484 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:13 smithi046 podman[76363]: 2024-04-30 10:01:13.283492362 +0000 UTC m=+0.525248289 container remove 6c6ff9e469ec996336638d396d6f67d304b7eefb2a47352927164bdcb218835b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:01:13.804 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:13 smithi046 podman[76430]: 2024-04-30 10:01:13.492764865 +0000 UTC m=+0.106381084 container create 501e0c080dadf56946460eb5ed4d00fc5bad879e3a1acbe48b28c5474c86a3b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424) 2024-04-30T10:01:13.804 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:13 smithi046 podman[76430]: 2024-04-30 10:01:13.405222016 +0000 UTC m=+0.018838237 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:01:13.804 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:13 smithi046 podman[76430]: 2024-04-30 10:01:13.712407556 +0000 UTC m=+0.326023783 container init 501e0c080dadf56946460eb5ed4d00fc5bad879e3a1acbe48b28c5474c86a3b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-30T10:01:13.804 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:13 smithi046 podman[76430]: 2024-04-30 10:01:13.715825662 +0000 UTC m=+0.329441883 container start 501e0c080dadf56946460eb5ed4d00fc5bad879e3a1acbe48b28c5474c86a3b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-04-30T10:01:13.805 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:13 smithi046 ceph-osd[76448]: -- 172.21.15.46:0/2004833818 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eb2a113680 con 0x55eb2a0dd000 2024-04-30T10:01:14.113 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:13 smithi046 bash[76430]: 501e0c080dadf56946460eb5ed4d00fc5bad879e3a1acbe48b28c5474c86a3b3 2024-04-30T10:01:14.113 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:13 smithi046 systemd[1]: Started Ceph osd.4 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:01:14.165 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:01:14.165 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (7m) 16s ago 29m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:01:14.165 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (6m) 4m ago 29m 78.3M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:01:14.165 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 16s ago 28m 62.3M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (8m) 4m ago 34m 450M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (18m) 16s ago 36m 508M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (4m) 16s ago 36m 53.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (5m) 4m ago 34m 35.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) 16s ago 34m 44.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (8m) 16s ago 30m 16.0M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (8m) 4m ago 30m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (2m) 16s ago 34m 48.3M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (2m) 16s ago 33m 44.4M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (76s) 16s ago 33m 41.5M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (21s) 16s ago 32m 15.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 8d72616b503d 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 starting - - - 3249M 2024-04-30T10:01:14.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (32m) 4m ago 32m 53.0M 3249M 17.2.0 e1d6a67b021e f883ecd1c5d1 2024-04-30T10:01:14.167 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (31m) 4m ago 31m 52.4M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T10:01:14.167 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (31m) 4m ago 31m 54.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T10:01:14.167 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (7m) 4m ago 30m 69.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:01:14.167 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (28m) 16s ago 28m 78.0M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T10:01:14.167 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (28m) 4m ago 28m 76.6M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T10:01:14.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[86310]: pgmap v191: 161 pgs: 17 active+undersized+degraded, 11 peering, 2 stale+active+clean, 31 active+undersized, 100 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[86310]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 17 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[86310]: from='client.44469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[80969]: pgmap v191: 161 pgs: 17 active+undersized+degraded, 11 peering, 2 stale+active+clean, 31 active+undersized, 100 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[80969]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 17 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[80969]: from='client.44469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:14.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:14.484 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:14 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:01:14.259+0000 7face1d7e7c0 -1 Falling back to public interface 2024-04-30T10:01:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:14 smithi046 ceph-mon[65689]: pgmap v191: 161 pgs: 17 active+undersized+degraded, 11 peering, 2 stale+active+clean, 31 active+undersized, 100 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 68/627 objects degraded (10.845%) 2024-04-30T10:01:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:14 smithi046 ceph-mon[65689]: Health check failed: Degraded data redundancy: 68/627 objects degraded (10.845%), 17 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:14 smithi046 ceph-mon[65689]: from='client.44469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:14.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:14.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 4 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:01:14.579 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T10:01:14.580 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T10:01:14.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:01:14.580 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T10:01:14.580 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 5, 2024-04-30T10:01:14.580 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 9 2024-04-30T10:01:14.580 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T10:01:14.580 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:01:14.933 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:01:14.933 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T10:01:14.933 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T10:01:14.933 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-30T10:01:14.933 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T10:01:14.933 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "4/8 daemons upgraded", 2024-04-30T10:01:14.933 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-30T10:01:14.934 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T10:01:14.934 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:01:15.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:15 smithi026 ceph-mon[86310]: from='client.44475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:15.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:15 smithi026 ceph-mon[86310]: from='client.54424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:15.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:15 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/3150355320' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:15.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:15 smithi026 ceph-mon[80969]: from='client.44475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:15.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:15 smithi026 ceph-mon[80969]: from='client.54424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:15.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:15 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/3150355320' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:15.444 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:15 smithi046 ceph-mon[65689]: from='client.44475 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:15.444 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:15 smithi046 ceph-mon[65689]: from='client.54424 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:15.444 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:15 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/3150355320' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:16.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:16 smithi046 ceph-mon[65689]: pgmap v192: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-30T10:01:16.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:16 smithi046 ceph-mon[65689]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:16.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:16 smithi046 ceph-mon[65689]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-04-30T10:01:16.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:16.385 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[86310]: pgmap v192: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[86310]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[86310]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[80969]: pgmap v192: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[80969]: from='client.44496 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[80969]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:16.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:01:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:01:16] "GET /metrics HTTP/1.1" 200 38184 "" "Prometheus/2.43.0" 2024-04-30T10:01:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:17 smithi046 ceph-mon[65689]: from='osd.4 [v2:172.21.15.46:6800/1568892810,v1:172.21.15.46:6801/1568892810]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T10:01:17.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:17 smithi046 ceph-mon[65689]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T10:01:17.234 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:01:16.999+0000 7face1d7e7c0 -1 osd.4 0 read_superblock omap replica is missing. 2024-04-30T10:01:17.234 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:01:17.029+0000 7face1d7e7c0 -1 osd.4 133 log_to_monitors true 2024-04-30T10:01:17.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:17 smithi026 ceph-mon[86310]: from='osd.4 [v2:172.21.15.46:6800/1568892810,v1:172.21.15.46:6801/1568892810]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T10:01:17.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:17 smithi026 ceph-mon[86310]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T10:01:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:17 smithi026 ceph-mon[80969]: from='osd.4 [v2:172.21.15.46:6800/1568892810,v1:172.21.15.46:6801/1568892810]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T10:01:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:17 smithi026 ceph-mon[80969]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-30T10:01:17.734 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:01:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:01:17.442+0000 7faccf1cd700 -1 osd.4 133 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T10:01:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[86310]: pgmap v193: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[86310]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[86310]: from='osd.4 [v2:172.21.15.46:6800/1568892810,v1:172.21.15.46:6801/1568892810]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[86310]: osdmap e136: 8 total, 7 up, 8 in 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[86310]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[86310]: from='osd.4 ' entity='osd.4' 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[80969]: pgmap v193: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[80969]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[80969]: from='osd.4 [v2:172.21.15.46:6800/1568892810,v1:172.21.15.46:6801/1568892810]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[80969]: osdmap e136: 8 total, 7 up, 8 in 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[80969]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:18 smithi026 ceph-mon[80969]: from='osd.4 ' entity='osd.4' 2024-04-30T10:01:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:18 smithi046 ceph-mon[65689]: pgmap v193: 161 pgs: 26 active+undersized+degraded, 42 active+undersized, 93 active+clean; 585 KiB data, 174 MiB used, 715 GiB / 715 GiB avail; 107/627 objects degraded (17.065%) 2024-04-30T10:01:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:18 smithi046 ceph-mon[65689]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-30T10:01:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:18 smithi046 ceph-mon[65689]: from='osd.4 [v2:172.21.15.46:6800/1568892810,v1:172.21.15.46:6801/1568892810]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:18 smithi046 ceph-mon[65689]: osdmap e136: 8 total, 7 up, 8 in 2024-04-30T10:01:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:18 smithi046 ceph-mon[65689]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:18 smithi046 ceph-mon[65689]: from='osd.4 ' entity='osd.4' 2024-04-30T10:01:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:19 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:19 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:19 smithi046 ceph-mon[65689]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:01:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:19 smithi046 ceph-mon[65689]: osd.4 [v2:172.21.15.46:6800/1568892810,v1:172.21.15.46:6801/1568892810] boot 2024-04-30T10:01:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:19 smithi046 ceph-mon[65689]: osdmap e137: 8 total, 8 up, 8 in 2024-04-30T10:01:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:19 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T10:01:19.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:19.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:19.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[86310]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:01:19.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[86310]: osd.4 [v2:172.21.15.46:6800/1568892810,v1:172.21.15.46:6801/1568892810] boot 2024-04-30T10:01:19.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[86310]: osdmap e137: 8 total, 8 up, 8 in 2024-04-30T10:01:19.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T10:01:19.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:19.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:19.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[80969]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:01:19.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[80969]: osd.4 [v2:172.21.15.46:6800/1568892810,v1:172.21.15.46:6801/1568892810] boot 2024-04-30T10:01:19.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[80969]: osdmap e137: 8 total, 8 up, 8 in 2024-04-30T10:01:19.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:19 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-30T10:01:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[86310]: pgmap v196: 161 pgs: 7 peering, 22 active+undersized+degraded, 39 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 90/627 objects degraded (14.354%) 2024-04-30T10:01:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 90/627 objects degraded (14.354%), 22 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:20.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[86310]: osdmap e138: 8 total, 8 up, 8 in 2024-04-30T10:01:20.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[80969]: pgmap v196: 161 pgs: 7 peering, 22 active+undersized+degraded, 39 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 90/627 objects degraded (14.354%) 2024-04-30T10:01:20.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:20.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:20.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 90/627 objects degraded (14.354%), 22 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:20.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:20 smithi026 ceph-mon[80969]: osdmap e138: 8 total, 8 up, 8 in 2024-04-30T10:01:20.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:20 smithi046 ceph-mon[65689]: pgmap v196: 161 pgs: 7 peering, 22 active+undersized+degraded, 39 active+undersized, 93 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 90/627 objects degraded (14.354%) 2024-04-30T10:01:20.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:20.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:20.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:20 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 90/627 objects degraded (14.354%), 22 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:20.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:20 smithi046 ceph-mon[65689]: osdmap e138: 8 total, 8 up, 8 in 2024-04-30T10:01:22.469 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:22 smithi046 ceph-mon[65689]: pgmap v198: 161 pgs: 7 peering, 21 active+undersized+degraded, 32 active+undersized, 101 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 83/627 objects degraded (13.238%) 2024-04-30T10:01:22.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:22 smithi026 ceph-mon[80969]: pgmap v198: 161 pgs: 7 peering, 21 active+undersized+degraded, 32 active+undersized, 101 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 83/627 objects degraded (13.238%) 2024-04-30T10:01:22.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:22 smithi026 ceph-mon[86310]: pgmap v198: 161 pgs: 7 peering, 21 active+undersized+degraded, 32 active+undersized, 101 active+clean; 585 KiB data, 185 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 83/627 objects degraded (13.238%) 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:23.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:23.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:23 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-30T10:01:23.708 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:23.708 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:23.709 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:23.709 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:23.709 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:01:23.709 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:23.709 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:23.709 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:23.709 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:23.709 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:23.709 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:23 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-30T10:01:24.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[86310]: pgmap v199: 161 pgs: 7 peering, 154 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-04-30T10:01:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[86310]: pgmap v200: 161 pgs: 7 peering, 154 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-04-30T10:01:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[86310]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-30T10:01:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[86310]: Upgrade: osd.5 is safe to restart 2024-04-30T10:01:24.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[86310]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 83/627 objects degraded (13.238%), 21 pgs degraded) 2024-04-30T10:01:24.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[80969]: pgmap v199: 161 pgs: 7 peering, 154 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-04-30T10:01:24.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[80969]: pgmap v200: 161 pgs: 7 peering, 154 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-04-30T10:01:24.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[80969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-30T10:01:24.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[80969]: Upgrade: osd.5 is safe to restart 2024-04-30T10:01:24.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:24 smithi026 ceph-mon[80969]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 83/627 objects degraded (13.238%), 21 pgs degraded) 2024-04-30T10:01:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:24 smithi046 ceph-mon[65689]: pgmap v199: 161 pgs: 7 peering, 154 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 862 B/s rd, 0 op/s 2024-04-30T10:01:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:24 smithi046 ceph-mon[65689]: pgmap v200: 161 pgs: 7 peering, 154 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 899 B/s rd, 0 op/s 2024-04-30T10:01:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:24 smithi046 ceph-mon[65689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["5"], "max": 16}]: dispatch 2024-04-30T10:01:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:24 smithi046 ceph-mon[65689]: Upgrade: osd.5 is safe to restart 2024-04-30T10:01:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:24 smithi046 ceph-mon[65689]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 83/627 objects degraded (13.238%), 21 pgs degraded) 2024-04-30T10:01:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:25 smithi046 ceph-mon[65689]: Upgrade: Updating osd.5 2024-04-30T10:01:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:25 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:25 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-30T10:01:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:25 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:25 smithi046 ceph-mon[65689]: Deploying daemon osd.5 on smithi046 2024-04-30T10:01:25.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[86310]: Upgrade: Updating osd.5 2024-04-30T10:01:25.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-30T10:01:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[86310]: Deploying daemon osd.5 on smithi046 2024-04-30T10:01:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[80969]: Upgrade: Updating osd.5 2024-04-30T10:01:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-30T10:01:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:25 smithi026 ceph-mon[80969]: Deploying daemon osd.5 on smithi046 2024-04-30T10:01:26.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:26 smithi026 ceph-mon[86310]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 802 B/s rd, 0 op/s 2024-04-30T10:01:26.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:26 smithi026 ceph-mon[80969]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 802 B/s rd, 0 op/s 2024-04-30T10:01:26.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:01:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:01:26] "GET /metrics HTTP/1.1" 200 38184 "" "Prometheus/2.43.0" 2024-04-30T10:01:26.702 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:26 smithi046 ceph-mon[65689]: pgmap v201: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 802 B/s rd, 0 op/s 2024-04-30T10:01:27.928 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:27 smithi046 systemd[1]: Stopping Ceph osd.5 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:01:28.234 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[29494]: 2024-04-30T10:01:27.926+0000 7fc2f48fb700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:01:28.234 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[29494]: 2024-04-30T10:01:27.926+0000 7fc2f48fb700 -1 osd.5 138 *** Got signal Terminated *** 2024-04-30T10:01:28.234 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[29494]: 2024-04-30T10:01:27.926+0000 7fc2f48fb700 -1 osd.5 138 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:01:28.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:28 smithi026 ceph-mon[86310]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T10:01:28.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:28 smithi026 ceph-mon[86310]: osd.5 marked itself down and dead 2024-04-30T10:01:28.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:28 smithi026 ceph-mon[80969]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T10:01:28.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:28 smithi026 ceph-mon[80969]: osd.5 marked itself down and dead 2024-04-30T10:01:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:28 smithi046 ceph-mon[65689]: pgmap v202: 161 pgs: 161 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T10:01:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:28 smithi046 ceph-mon[65689]: osd.5 marked itself down and dead 2024-04-30T10:01:29.296 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:29 smithi046 podman[80562]: 2024-04-30 10:01:29.034127749 +0000 UTC m=+1.232147406 container died f883ecd1c5d19af05bceb1b88ea6eb1557a6a65081deb149f7eeae6f5fea537a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, GIT_BRANCH=HEAD, GIT_CLEAN=True, vcs-type=git, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.component=centos-stream-container, distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, architecture=x86_64, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-30T10:01:29.679 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:29 smithi046 ceph-mon[65689]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:01:29.679 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:29 smithi046 ceph-mon[65689]: osdmap e139: 8 total, 7 up, 8 in 2024-04-30T10:01:29.680 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:29 smithi046 ceph-mon[65689]: osdmap e140: 8 total, 7 up, 8 in 2024-04-30T10:01:29.680 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:29 smithi046 podman[80562]: 2024-04-30 10:01:29.402115264 +0000 UTC m=+1.600134916 container cleanup f883ecd1c5d19af05bceb1b88ea6eb1557a6a65081deb149f7eeae6f5fea537a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5, ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, name=centos-stream, distribution-scope=public, vcs-type=git, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, architecture=x86_64, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., io.buildah.version=1.19.8, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, version=8, release=754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-30T10:01:29.680 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:29 smithi046 bash[80562]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5 2024-04-30T10:01:29.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:29 smithi026 ceph-mon[86310]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:01:29.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:29 smithi026 ceph-mon[86310]: osdmap e139: 8 total, 7 up, 8 in 2024-04-30T10:01:29.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:29 smithi026 ceph-mon[86310]: osdmap e140: 8 total, 7 up, 8 in 2024-04-30T10:01:29.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:29 smithi026 ceph-mon[80969]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:01:29.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:29 smithi026 ceph-mon[80969]: osdmap e139: 8 total, 7 up, 8 in 2024-04-30T10:01:29.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:29 smithi026 ceph-mon[80969]: osdmap e140: 8 total, 7 up, 8 in 2024-04-30T10:01:29.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:29 smithi046 podman[80583]: 2024-04-30 10:01:29.678843538 +0000 UTC m=+0.646488328 container remove f883ecd1c5d19af05bceb1b88ea6eb1557a6a65081deb149f7eeae6f5fea537a (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, io.openshift.expose-services=, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, build-date=2022-05-03T08:36:31.336870, ceph=True, RELEASE=HEAD, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, CEPH_POINT_RELEASE=-17.2.0, GIT_REPO=https://github.com/ceph/ceph-container.git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., vcs-type=git, release=754, maintainer=Guillaume Abrioux , vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.buildah.version=1.19.8, architecture=x86_64) 2024-04-30T10:01:30.334 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:30 smithi046 podman[80674]: 2024-04-30 10:01:30.0380705 +0000 UTC m=+0.104638467 container create b20793b77ae5e499a44af5e8203fb77a5bc275c02b7317b50fdfb68f76601eea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, ceph=True) 2024-04-30T10:01:30.334 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:30 smithi046 podman[80674]: 2024-04-30 10:01:29.952191275 +0000 UTC m=+0.018759215 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:01:30.334 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:30 smithi046 podman[80674]: 2024-04-30 10:01:30.223552985 +0000 UTC m=+0.290120907 container init b20793b77ae5e499a44af5e8203fb77a5bc275c02b7317b50fdfb68f76601eea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:01:30.334 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:30 smithi046 podman[80674]: 2024-04-30 10:01:30.226819558 +0000 UTC m=+0.293387487 container start b20793b77ae5e499a44af5e8203fb77a5bc275c02b7317b50fdfb68f76601eea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424) 2024-04-30T10:01:30.334 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:30 smithi046 podman[80674]: 2024-04-30 10:01:30.333563368 +0000 UTC m=+0.400131290 container attach b20793b77ae5e499a44af5e8203fb77a5bc275c02b7317b50fdfb68f76601eea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5) 2024-04-30T10:01:30.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:30 smithi046 ceph-mon[65689]: pgmap v204: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T10:01:30.735 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:30 smithi046 podman[80674]: 2024-04-30 10:01:30.534983416 +0000 UTC m=+0.601551352 container died b20793b77ae5e499a44af5e8203fb77a5bc275c02b7317b50fdfb68f76601eea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:01:30.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:30 smithi026 ceph-mon[86310]: pgmap v204: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T10:01:30.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:30 smithi026 ceph-mon[80969]: pgmap v204: 161 pgs: 21 stale+active+clean, 140 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s rd, 1 op/s 2024-04-30T10:01:31.484 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 podman[80700]: 2024-04-30 10:01:31.118852765 +0000 UTC m=+0.575097992 container remove b20793b77ae5e499a44af5e8203fb77a5bc275c02b7317b50fdfb68f76601eea (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS) 2024-04-30T10:01:31.484 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-30T10:01:31.484 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 systemd[1]: Stopped Ceph osd.5 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:01:31.485 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.5.service: Consumed 17.224s CPU time. 2024-04-30T10:01:31.485 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 systemd[1]: Starting Ceph osd.5 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:01:31.906 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:31.907 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:31.907 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:31 smithi046 ceph-mon[65689]: pgmap v206: 161 pgs: 2 active+undersized+degraded, 6 peering, 15 stale+active+clean, 10 active+undersized, 128 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 11/627 objects degraded (1.754%) 2024-04-30T10:01:31.907 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:31.907 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 podman[80808]: 2024-04-30 10:01:31.47470181 +0000 UTC m=+0.019248276 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:01:31.907 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 podman[80808]: 2024-04-30 10:01:31.578656192 +0000 UTC m=+0.123202646 container create 42699aaadb6cb2f9a591ed8dbc6d2b39e99dc1dc2392b3592325008534760504 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-04-30T10:01:31.907 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 podman[80808]: 2024-04-30 10:01:31.815595884 +0000 UTC m=+0.360142334 container init 42699aaadb6cb2f9a591ed8dbc6d2b39e99dc1dc2392b3592325008534760504 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:01:31.907 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 podman[80808]: 2024-04-30 10:01:31.819059925 +0000 UTC m=+0.363606375 container start 42699aaadb6cb2f9a591ed8dbc6d2b39e99dc1dc2392b3592325008534760504 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:01:31.907 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:31 smithi046 podman[80808]: 2024-04-30 10:01:31.906224298 +0000 UTC m=+0.450770761 container attach 42699aaadb6cb2f9a591ed8dbc6d2b39e99dc1dc2392b3592325008534760504 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0) 2024-04-30T10:01:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:31 smithi026 ceph-mon[86310]: pgmap v206: 161 pgs: 2 active+undersized+degraded, 6 peering, 15 stale+active+clean, 10 active+undersized, 128 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 11/627 objects degraded (1.754%) 2024-04-30T10:01:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:31 smithi026 ceph-mon[80969]: pgmap v206: 161 pgs: 2 active+undersized+degraded, 6 peering, 15 stale+active+clean, 10 active+undersized, 128 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 11/627 objects degraded (1.754%) 2024-04-30T10:01:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:32.635 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate[80823]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate[80823]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 bash[80808]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 bash[80808]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 bash[80808]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 bash[80808]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 bash[80808]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 bash[80808]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 bash[80808]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 bash[80808]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 bash[80808]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate[80823]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate[80823]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-5 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate[80823]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate[80823]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-30T10:01:32.636 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate[80823]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-5/block 2024-04-30T10:01:32.637 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate[80823]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2024-04-30T10:01:32.637 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate[80823]: --> ceph-volume raw activate successful for osd ID: 5 2024-04-30T10:01:32.637 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 podman[80808]: 2024-04-30 10:01:32.337503012 +0000 UTC m=+0.882049464 container died 42699aaadb6cb2f9a591ed8dbc6d2b39e99dc1dc2392b3592325008534760504 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:01:32.944 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-mon[65689]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-30T10:01:32.944 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-mon[65689]: Health check failed: Degraded data redundancy: 11/627 objects degraded (1.754%), 2 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:32.944 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:32 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:33.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:32 smithi026 ceph-mon[80969]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-30T10:01:33.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:32 smithi026 ceph-mon[80969]: Health check failed: Degraded data redundancy: 11/627 objects degraded (1.754%), 2 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:32 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:32 smithi026 ceph-mon[86310]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-30T10:01:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:32 smithi026 ceph-mon[86310]: Health check failed: Degraded data redundancy: 11/627 objects degraded (1.754%), 2 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:32 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:33.234 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:32 smithi046 podman[80955]: 2024-04-30 10:01:32.943450709 +0000 UTC m=+0.600731337 container remove 42699aaadb6cb2f9a591ed8dbc6d2b39e99dc1dc2392b3592325008534760504 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-activate, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-04-30T10:01:33.234 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:33 smithi046 podman[81021]: 2024-04-30 10:01:33.076664374 +0000 UTC m=+0.019463816 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:01:33.234 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:33 smithi046 podman[81021]: 2024-04-30 10:01:33.177794093 +0000 UTC m=+0.120593525 container create b070c14531b045b8698b4eba2907d9d7daed539f4e24a4a38ba894ff963e517b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:01:33.734 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:33 smithi046 podman[81021]: 2024-04-30 10:01:33.38003996 +0000 UTC m=+0.322839392 container init b070c14531b045b8698b4eba2907d9d7daed539f4e24a4a38ba894ff963e517b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:01:33.734 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:33 smithi046 podman[81021]: 2024-04-30 10:01:33.383346389 +0000 UTC m=+0.326145820 container start b070c14531b045b8698b4eba2907d9d7daed539f4e24a4a38ba894ff963e517b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5, org.label-schema.build-date=20240424, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-04-30T10:01:33.734 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:33 smithi046 ceph-osd[81040]: -- 172.21.15.46:0/3575573973 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55f762397680 con 0x55f762361000 2024-04-30T10:01:33.734 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:33 smithi046 bash[81021]: b070c14531b045b8698b4eba2907d9d7daed539f4e24a4a38ba894ff963e517b 2024-04-30T10:01:33.734 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:33 smithi046 systemd[1]: Started Ceph osd.5 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:01:34.234 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:33 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:01:33.924+0000 7f63e34257c0 -1 Falling back to public interface 2024-04-30T10:01:34.583 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:34 smithi046 ceph-mon[65689]: pgmap v207: 161 pgs: 17 active+undersized+degraded, 6 peering, 36 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 63/627 objects degraded (10.048%) 2024-04-30T10:01:34.584 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:34 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:34.584 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:34 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:34.584 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:34 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:34.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:34 smithi026 ceph-mon[86310]: pgmap v207: 161 pgs: 17 active+undersized+degraded, 6 peering, 36 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 63/627 objects degraded (10.048%) 2024-04-30T10:01:34.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:34 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:34.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:34 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:34.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:34 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:34 smithi026 ceph-mon[80969]: pgmap v207: 161 pgs: 17 active+undersized+degraded, 6 peering, 36 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 63/627 objects degraded (10.048%) 2024-04-30T10:01:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:34 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:34 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:34.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:34 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:35.889 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:35 smithi046 ceph-mon[65689]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-30T10:01:35.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:35 smithi026 ceph-mon[80969]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-30T10:01:35.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:35 smithi026 ceph-mon[86310]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-30T10:01:36.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:36 smithi026 ceph-mon[80969]: pgmap v208: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-30T10:01:36.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:36 smithi026 ceph-mon[86310]: pgmap v208: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-30T10:01:36.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:01:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:01:36] "GET /metrics HTTP/1.1" 200 38192 "" "Prometheus/2.43.0" 2024-04-30T10:01:36.859 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:36 smithi046 ceph-mon[65689]: pgmap v208: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-30T10:01:36.860 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:36 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:01:36.643+0000 7f63e34257c0 -1 osd.5 0 read_superblock omap replica is missing. 2024-04-30T10:01:36.860 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:36 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:01:36.669+0000 7f63e34257c0 -1 osd.5 138 log_to_monitors true 2024-04-30T10:01:37.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:37 smithi046 ceph-mon[65689]: from='osd.5 [v2:172.21.15.46:6808/3336280182,v1:172.21.15.46:6809/3336280182]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T10:01:37.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:37 smithi046 ceph-mon[65689]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T10:01:37.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:37 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 73/627 objects degraded (11.643%), 21 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:37.734 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:01:37 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:01:37.576+0000 7f63d487c700 -1 osd.5 138 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T10:01:37.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:37 smithi026 ceph-mon[86310]: from='osd.5 [v2:172.21.15.46:6808/3336280182,v1:172.21.15.46:6809/3336280182]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T10:01:37.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:37 smithi026 ceph-mon[86310]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T10:01:37.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:37 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 73/627 objects degraded (11.643%), 21 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:37.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:37 smithi026 ceph-mon[80969]: from='osd.5 [v2:172.21.15.46:6808/3336280182,v1:172.21.15.46:6809/3336280182]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T10:01:37.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:37 smithi026 ceph-mon[80969]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-30T10:01:37.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:37 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 73/627 objects degraded (11.643%), 21 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[86310]: pgmap v209: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[86310]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[86310]: from='osd.5 [v2:172.21.15.46:6808/3336280182,v1:172.21.15.46:6809/3336280182]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[86310]: osdmap e141: 8 total, 7 up, 8 in 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[86310]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[80969]: pgmap v209: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[80969]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-30T10:01:38.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[80969]: from='osd.5 [v2:172.21.15.46:6808/3336280182,v1:172.21.15.46:6809/3336280182]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:38.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[80969]: osdmap e141: 8 total, 7 up, 8 in 2024-04-30T10:01:38.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[80969]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:38.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:38.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:38 smithi046 ceph-mon[65689]: pgmap v209: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 190 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-30T10:01:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:38 smithi046 ceph-mon[65689]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-30T10:01:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:38 smithi046 ceph-mon[65689]: from='osd.5 [v2:172.21.15.46:6808/3336280182,v1:172.21.15.46:6809/3336280182]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:38 smithi046 ceph-mon[65689]: osdmap e141: 8 total, 7 up, 8 in 2024-04-30T10:01:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:38 smithi046 ceph-mon[65689]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:01:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:38.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:39.870 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:39 smithi046 ceph-mon[65689]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:01:39.870 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:39 smithi046 ceph-mon[65689]: osd.5 [v2:172.21.15.46:6808/3336280182,v1:172.21.15.46:6809/3336280182] boot 2024-04-30T10:01:39.870 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:39 smithi046 ceph-mon[65689]: osdmap e142: 8 total, 8 up, 8 in 2024-04-30T10:01:39.870 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T10:01:39.870 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:39.870 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:39.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[86310]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[86310]: osd.5 [v2:172.21.15.46:6808/3336280182,v1:172.21.15.46:6809/3336280182] boot 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[86310]: osdmap e142: 8 total, 8 up, 8 in 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[80969]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[80969]: osd.5 [v2:172.21.15.46:6808/3336280182,v1:172.21.15.46:6809/3336280182] boot 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[80969]: osdmap e142: 8 total, 8 up, 8 in 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:39.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:40 smithi046 ceph-mon[65689]: pgmap v212: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-30T10:01:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:40 smithi046 ceph-mon[65689]: osdmap e143: 8 total, 8 up, 8 in 2024-04-30T10:01:40.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:40 smithi026 ceph-mon[86310]: pgmap v212: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-30T10:01:40.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:40 smithi026 ceph-mon[86310]: osdmap e143: 8 total, 8 up, 8 in 2024-04-30T10:01:40.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:40 smithi026 ceph-mon[80969]: pgmap v212: 161 pgs: 21 active+undersized+degraded, 38 active+undersized, 102 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 73/627 objects degraded (11.643%) 2024-04-30T10:01:40.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:40 smithi026 ceph-mon[80969]: osdmap e143: 8 total, 8 up, 8 in 2024-04-30T10:01:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:42 smithi046 ceph-mon[65689]: pgmap v214: 161 pgs: 5 peering, 16 active+undersized+degraded, 31 active+undersized, 109 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 53/627 objects degraded (8.453%) 2024-04-30T10:01:42.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:42 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 53/627 objects degraded (8.453%), 16 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:42.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:42 smithi026 ceph-mon[86310]: pgmap v214: 161 pgs: 5 peering, 16 active+undersized+degraded, 31 active+undersized, 109 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 53/627 objects degraded (8.453%) 2024-04-30T10:01:42.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:42 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 53/627 objects degraded (8.453%), 16 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:42.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:42 smithi026 ceph-mon[80969]: pgmap v214: 161 pgs: 5 peering, 16 active+undersized+degraded, 31 active+undersized, 109 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 53/627 objects degraded (8.453%) 2024-04-30T10:01:42.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:42 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 53/627 objects degraded (8.453%), 16 pgs degraded (PG_DEGRADED) 2024-04-30T10:01:43.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T10:01:43.005+0000 7f3e7839e700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-30T10:01:43.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:43.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:43.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:43 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:43.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:43.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:43.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:01:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:01:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:01:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:43.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:43 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:44.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[86310]: pgmap v215: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[86310]: pgmap v216: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[86310]: pgmap v217: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 714 B/s rd, 0 op/s 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[86310]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[86310]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[86310]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 53/627 objects degraded (8.453%), 16 pgs degraded) 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[80969]: pgmap v215: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[80969]: pgmap v216: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[80969]: pgmap v217: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 714 B/s rd, 0 op/s 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[80969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[80969]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-30T10:01:44.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:44 smithi026 ceph-mon[80969]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 53/627 objects degraded (8.453%), 16 pgs degraded) 2024-04-30T10:01:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:44 smithi046 ceph-mon[65689]: pgmap v215: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s 2024-04-30T10:01:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:44 smithi046 ceph-mon[65689]: pgmap v216: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 578 B/s rd, 0 op/s 2024-04-30T10:01:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:44 smithi046 ceph-mon[65689]: pgmap v217: 161 pgs: 5 peering, 156 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 714 B/s rd, 0 op/s 2024-04-30T10:01:44.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:44 smithi046 ceph-mon[65689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:44.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:44 smithi046 ceph-mon[65689]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-04-30T10:01:44.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:44 smithi046 ceph-mon[65689]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 53/627 objects degraded (8.453%), 16 pgs degraded) 2024-04-30T10:01:45.304 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:01:45.995 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:01:45.995 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (7m) 47s ago 30m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:01:45.995 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (7m) 8s ago 30m 79.7M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:01:45.995 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 47s ago 29m 62.3M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:01:45.995 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (9m) 8s ago 34m 455M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:01:45.995 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (19m) 47s ago 36m 508M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:01:45.995 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (5m) 47s ago 36m 53.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:01:45.995 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (6m) 8s ago 35m 46.1M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:01:45.995 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) 47s ago 35m 44.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (8m) 47s ago 30m 16.0M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (8m) 8s ago 30m 17.2M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (3m) 47s ago 34m 48.3M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (2m) 47s ago 34m 44.4M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (108s) 47s ago 33m 41.5M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (52s) 47s ago 33m 15.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 8d72616b503d 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (32s) 8s ago 32m 50.3M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a 501e0c080dad 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (12s) 8s ago 32m 12.6M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b070c14531b0 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (32m) 8s ago 32m 56.9M 3249M 17.2.0 e1d6a67b021e 56cf351755f6 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (31m) 8s ago 31m 60.6M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (8m) 8s ago 31m 80.2M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (29m) 47s ago 29m 78.0M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T10:01:45.996 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (29m) 8s ago 29m 78.2M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T10:01:46.416 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:01:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T10:01:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T10:01:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:01:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T10:01:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 6 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 4, 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 11 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T10:01:46.417 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:01:46.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[86310]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 509 B/s rd, 0 op/s 2024-04-30T10:01:46.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[86310]: from='client.54445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:46.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:46.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:46.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2802321456' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:46.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:01:46] "GET /metrics HTTP/1.1" 200 38199 "" "Prometheus/2.43.0" 2024-04-30T10:01:46.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[80969]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 509 B/s rd, 0 op/s 2024-04-30T10:01:46.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[80969]: from='client.54445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:46.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:46.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:46.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:46 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2802321456' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:46.770 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:01:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T10:01:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T10:01:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-30T10:01:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T10:01:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "6/8 daemons upgraded", 2024-04-30T10:01:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-30T10:01:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T10:01:46.772 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:01:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:46 smithi046 ceph-mon[65689]: pgmap v218: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 509 B/s rd, 0 op/s 2024-04-30T10:01:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:46 smithi046 ceph-mon[65689]: from='client.54445 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:01:46.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:46 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2802321456' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:01:47.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:47 smithi026 ceph-mon[80969]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:47.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:47 smithi026 ceph-mon[80969]: from='client.54457 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:47 smithi026 ceph-mon[86310]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:47.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:47 smithi026 ceph-mon[86310]: from='client.54457 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:47.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:47 smithi046 ceph-mon[65689]: from='client.44514 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:47.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:47 smithi046 ceph-mon[65689]: from='client.54457 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:48.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:48 smithi026 ceph-mon[80969]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:48.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:48 smithi026 ceph-mon[80969]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:01:48.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:48 smithi026 ceph-mon[86310]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:48.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:48 smithi026 ceph-mon[86310]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:01:48.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:48 smithi046 ceph-mon[65689]: from='client.54469 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:01:48.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:48 smithi046 ceph-mon[65689]: pgmap v219: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:01:50.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:50 smithi026 ceph-mon[86310]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-04-30T10:01:50.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:50 smithi026 ceph-mon[80969]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-04-30T10:01:50.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:50 smithi046 ceph-mon[65689]: pgmap v220: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1019 B/s rd, 0 op/s 2024-04-30T10:01:52.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:52 smithi026 ceph-mon[86310]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-04-30T10:01:52.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:52 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:52.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:52 smithi026 ceph-mon[80969]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-04-30T10:01:52.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:52 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:52.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:52 smithi046 ceph-mon[65689]: pgmap v221: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 637 B/s rd, 0 op/s 2024-04-30T10:01:52.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:52 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:01:53.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:53 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:53 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:53 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:01:54.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:54 smithi026 ceph-mon[86310]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:01:54.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:54 smithi026 ceph-mon[80969]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:01:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:54 smithi046 ceph-mon[65689]: pgmap v222: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:01:56.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:01:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:01:56] "GET /metrics HTTP/1.1" 200 38199 "" "Prometheus/2.43.0" 2024-04-30T10:01:56.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:56 smithi026 ceph-mon[86310]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:01:56.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:56 smithi026 ceph-mon[80969]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:01:56.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:56 smithi046 ceph-mon[65689]: pgmap v223: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:01:58.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:58 smithi026 ceph-mon[86310]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:01:58.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:58 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:58.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:58 smithi026 ceph-mon[80969]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:01:58.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:58 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:58 smithi046 ceph-mon[65689]: pgmap v224: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:01:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:58 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:59.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:59 smithi046 ceph-mon[65689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:59.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:01:59 smithi046 ceph-mon[65689]: Upgrade: osd.6 is safe to restart 2024-04-30T10:01:59.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:59 smithi026 ceph-mon[86310]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:01:59 smithi026 ceph-mon[86310]: Upgrade: osd.6 is safe to restart 2024-04-30T10:01:59.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:59 smithi026 ceph-mon[80969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["6"], "max": 16}]: dispatch 2024-04-30T10:01:59.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:01:59 smithi026 ceph-mon[80969]: Upgrade: osd.6 is safe to restart 2024-04-30T10:02:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:00 smithi046 ceph-mon[65689]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:00 smithi046 ceph-mon[65689]: Upgrade: Updating osd.6 2024-04-30T10:02:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-30T10:02:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:00.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[80969]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:00.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[80969]: Upgrade: Updating osd.6 2024-04-30T10:02:00.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:00.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-30T10:02:00.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[86310]: pgmap v225: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[86310]: Upgrade: Updating osd.6 2024-04-30T10:02:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-30T10:02:00.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:01.896 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:01 smithi046 ceph-mon[65689]: Deploying daemon osd.6 on smithi046 2024-04-30T10:02:01.896 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:01.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:01 smithi026 ceph-mon[86310]: Deploying daemon osd.6 on smithi046 2024-04-30T10:02:01.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:01.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:01 smithi026 ceph-mon[80969]: Deploying daemon osd.6 on smithi046 2024-04-30T10:02:01.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:02.942 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:02 smithi046 ceph-mon[65689]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:02.942 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:02 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:02.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:02 smithi026 ceph-mon[86310]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:02.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:02 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:02.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:02 smithi026 ceph-mon[80969]: pgmap v226: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:02.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:02 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:03.204 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:02 smithi046 systemd[1]: Stopping Ceph osd.6 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:02:03.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[32379]: 2024-04-30T10:02:03.201+0000 7f6ac0e4e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:02:03.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[32379]: 2024-04-30T10:02:03.201+0000 7f6ac0e4e700 -1 osd.6 143 *** Got signal Terminated *** 2024-04-30T10:02:03.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[32379]: 2024-04-30T10:02:03.201+0000 7f6ac0e4e700 -1 osd.6 143 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:02:03.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:03 smithi026 ceph-mon[86310]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:03.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:03 smithi026 ceph-mon[86310]: osd.6 marked itself down and dead 2024-04-30T10:02:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:03 smithi026 ceph-mon[80969]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:03.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:03 smithi026 ceph-mon[80969]: osd.6 marked itself down and dead 2024-04-30T10:02:03.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:03 smithi046 ceph-mon[65689]: pgmap v227: 161 pgs: 161 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:03.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:03 smithi046 ceph-mon[65689]: osd.6 marked itself down and dead 2024-04-30T10:02:04.294 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:04 smithi046 podman[85557]: 2024-04-30 10:02:04.018009726 +0000 UTC m=+0.928188910 container died 56cf351755f603e9726e8f1b9f2b5c823dc561b058031ba5ae58014b97887589 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6, architecture=x86_64, vcs-type=git, release=754, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, ceph=True, GIT_CLEAN=True, name=centos-stream, CEPH_POINT_RELEASE=-17.2.0, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, io.openshift.tags=base centos centos-stream, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., version=8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-30T10:02:04.608 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:04 smithi046 podman[85557]: 2024-04-30 10:02:04.294316059 +0000 UTC m=+1.204495248 container cleanup 56cf351755f603e9726e8f1b9f2b5c823dc561b058031ba5ae58014b97887589 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6, distribution-scope=public, io.openshift.expose-services=, name=centos-stream, GIT_BRANCH=HEAD, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.k8s.display-name=CentOS Stream 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, version=8, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, architecture=x86_64, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.tags=base centos centos-stream, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.buildah.version=1.19.8, ceph=True, GIT_CLEAN=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, build-date=2022-05-03T08:36:31.336870) 2024-04-30T10:02:04.608 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:04 smithi046 bash[85557]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6 2024-04-30T10:02:04.608 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:04 smithi046 podman[85574]: 2024-04-30 10:02:04.528800252 +0000 UTC m=+0.510600832 container remove 56cf351755f603e9726e8f1b9f2b5c823dc561b058031ba5ae58014b97887589 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6, release=754, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-type=git, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_REPO=https://github.com/ceph/ceph-container.git, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_CLEAN=True, version=8, io.k8s.display-name=CentOS Stream 8, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., build-date=2022-05-03T08:36:31.336870, io.buildah.version=1.19.8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, architecture=x86_64, maintainer=Guillaume Abrioux , io.openshift.expose-services=, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, RELEASE=HEAD) 2024-04-30T10:02:04.859 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:04 smithi046 ceph-mon[65689]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:02:04.859 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:04 smithi046 ceph-mon[65689]: osdmap e144: 8 total, 7 up, 8 in 2024-04-30T10:02:04.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:04 smithi026 ceph-mon[86310]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:02:04.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:04 smithi026 ceph-mon[86310]: osdmap e144: 8 total, 7 up, 8 in 2024-04-30T10:02:04.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:04 smithi026 ceph-mon[80969]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:02:04.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:04 smithi026 ceph-mon[80969]: osdmap e144: 8 total, 7 up, 8 in 2024-04-30T10:02:05.116 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:04 smithi046 podman[85664]: 2024-04-30 10:02:04.762062743 +0000 UTC m=+0.018864517 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:02:05.116 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:04 smithi046 podman[85664]: 2024-04-30 10:02:04.896518263 +0000 UTC m=+0.153320095 container create 47cddbf7a334983a6f2d3ba149c9ccddf71dea393d69172ab299dcb1118dfeb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-30T10:02:05.116 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:05 smithi046 podman[85664]: 2024-04-30 10:02:05.115748787 +0000 UTC m=+0.372550548 container init 47cddbf7a334983a6f2d3ba149c9ccddf71dea393d69172ab299dcb1118dfeb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-30T10:02:05.424 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:05 smithi046 podman[85664]: 2024-04-30 10:02:05.119017254 +0000 UTC m=+0.375819014 container start 47cddbf7a334983a6f2d3ba149c9ccddf71dea393d69172ab299dcb1118dfeb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:02:05.424 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:05 smithi046 podman[85664]: 2024-04-30 10:02:05.232320901 +0000 UTC m=+0.489122665 container attach 47cddbf7a334983a6f2d3ba149c9ccddf71dea393d69172ab299dcb1118dfeb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:02:05.714 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:05 smithi046 ceph-mon[65689]: osdmap e145: 8 total, 7 up, 8 in 2024-04-30T10:02:05.714 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:05 smithi046 ceph-mon[65689]: pgmap v230: 161 pgs: 1 active+undersized+degraded, 3 peering, 10 stale+active+clean, 4 active+undersized, 143 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 9/627 objects degraded (1.435%) 2024-04-30T10:02:05.715 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:05 smithi046 podman[85664]: 2024-04-30 10:02:05.424085315 +0000 UTC m=+0.680887092 container died 47cddbf7a334983a6f2d3ba149c9ccddf71dea393d69172ab299dcb1118dfeb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux ) 2024-04-30T10:02:05.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:05 smithi026 ceph-mon[86310]: osdmap e145: 8 total, 7 up, 8 in 2024-04-30T10:02:05.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:05 smithi026 ceph-mon[86310]: pgmap v230: 161 pgs: 1 active+undersized+degraded, 3 peering, 10 stale+active+clean, 4 active+undersized, 143 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 9/627 objects degraded (1.435%) 2024-04-30T10:02:05.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:05 smithi026 ceph-mon[80969]: osdmap e145: 8 total, 7 up, 8 in 2024-04-30T10:02:05.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:05 smithi026 ceph-mon[80969]: pgmap v230: 161 pgs: 1 active+undersized+degraded, 3 peering, 10 stale+active+clean, 4 active+undersized, 143 active+clean; 585 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 0 op/s; 9/627 objects degraded (1.435%) 2024-04-30T10:02:06.446 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:05 smithi046 podman[85690]: 2024-04-30 10:02:05.987810252 +0000 UTC m=+0.558127672 container remove 47cddbf7a334983a6f2d3ba149c9ccddf71dea393d69172ab299dcb1118dfeb8 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:02:06.446 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:06 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.6.service: Deactivated successfully. 2024-04-30T10:02:06.446 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:06 smithi046 systemd[1]: Stopped Ceph osd.6 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:02:06.446 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:06 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.6.service: Consumed 15.572s CPU time. 2024-04-30T10:02:06.446 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:06 smithi046 systemd[1]: Starting Ceph osd.6 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:02:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:02:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:02:06] "GET /metrics HTTP/1.1" 200 38203 "" "Prometheus/2.43.0" 2024-04-30T10:02:06.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:06 smithi026 ceph-mon[86310]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-30T10:02:06.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:06 smithi026 ceph-mon[86310]: Health check failed: Degraded data redundancy: 9/627 objects degraded (1.435%), 1 pg degraded (PG_DEGRADED) 2024-04-30T10:02:06.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:06 smithi026 ceph-mon[80969]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-30T10:02:06.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:06 smithi026 ceph-mon[80969]: Health check failed: Degraded data redundancy: 9/627 objects degraded (1.435%), 1 pg degraded (PG_DEGRADED) 2024-04-30T10:02:06.708 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:06 smithi046 ceph-mon[65689]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-30T10:02:06.708 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:06 smithi046 ceph-mon[65689]: Health check failed: Degraded data redundancy: 9/627 objects degraded (1.435%), 1 pg degraded (PG_DEGRADED) 2024-04-30T10:02:06.709 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:06 smithi046 podman[85797]: 2024-04-30 10:02:06.34799241 +0000 UTC m=+0.018442300 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:02:06.709 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:06 smithi046 podman[85797]: 2024-04-30 10:02:06.44707656 +0000 UTC m=+0.117526442 container create 71ee50e5915344acc6d50efa7f8a5456df56032fc34f58c45ecd488db38d8ab4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-30T10:02:06.984 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:06 smithi046 podman[85797]: 2024-04-30 10:02:06.708025944 +0000 UTC m=+0.378475825 container init 71ee50e5915344acc6d50efa7f8a5456df56032fc34f58c45ecd488db38d8ab4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-30T10:02:06.984 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:06 smithi046 podman[85797]: 2024-04-30 10:02:06.7118499 +0000 UTC m=+0.382299781 container start 71ee50e5915344acc6d50efa7f8a5456df56032fc34f58c45ecd488db38d8ab4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424) 2024-04-30T10:02:06.984 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:06 smithi046 podman[85797]: 2024-04-30 10:02:06.799789272 +0000 UTC m=+0.470239160 container attach 71ee50e5915344acc6d50efa7f8a5456df56032fc34f58c45ecd488db38d8ab4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:02:07.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate[85812]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:02:07.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate[85812]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:02:07.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 bash[85797]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:02:07.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 bash[85797]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:02:07.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 bash[85797]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-30T10:02:07.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 bash[85797]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-30T10:02:07.486 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 bash[85797]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-30T10:02:07.486 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 bash[85797]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-30T10:02:07.486 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 bash[85797]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-30T10:02:07.486 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 bash[85797]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-30T10:02:07.486 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 bash[85797]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-30T10:02:07.486 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate[85812]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-30T10:02:07.487 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate[85812]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-6 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-30T10:02:07.487 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate[85812]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-30T10:02:07.487 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate[85812]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-30T10:02:07.487 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate[85812]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-6/block 2024-04-30T10:02:07.487 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate[85812]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2024-04-30T10:02:07.487 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate[85812]: --> ceph-volume raw activate successful for osd ID: 6 2024-04-30T10:02:07.487 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 podman[85797]: 2024-04-30 10:02:07.23429998 +0000 UTC m=+0.904749862 container died 71ee50e5915344acc6d50efa7f8a5456df56032fc34f58c45ecd488db38d8ab4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-30T10:02:08.091 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:07 smithi046 podman[85940]: 2024-04-30 10:02:07.803899123 +0000 UTC m=+0.564366935 container remove 71ee50e5915344acc6d50efa7f8a5456df56032fc34f58c45ecd488db38d8ab4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-activate, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-30T10:02:08.091 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:08 smithi046 podman[86005]: 2024-04-30 10:02:07.926949307 +0000 UTC m=+0.019154388 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:02:08.091 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:08 smithi046 podman[86005]: 2024-04-30 10:02:08.038316217 +0000 UTC m=+0.130521306 container create ce35c1eb8525dffd057b272472473105ce91bc83cf36d765bdfcf4da931ab120 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424) 2024-04-30T10:02:08.374 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:08 smithi046 ceph-mon[65689]: pgmap v231: 161 pgs: 9 active+undersized+degraded, 3 peering, 8 stale+active+clean, 17 active+undersized, 124 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 37/627 objects degraded (5.901%) 2024-04-30T10:02:08.374 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:08 smithi046 podman[86005]: 2024-04-30 10:02:08.273936281 +0000 UTC m=+0.366141328 container init ce35c1eb8525dffd057b272472473105ce91bc83cf36d765bdfcf4da931ab120 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-04-30T10:02:08.374 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:08 smithi046 podman[86005]: 2024-04-30 10:02:08.277005845 +0000 UTC m=+0.369210897 container start ce35c1eb8525dffd057b272472473105ce91bc83cf36d765bdfcf4da931ab120 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True) 2024-04-30T10:02:08.374 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:08 smithi046 ceph-osd[86025]: -- 172.21.15.46:0/3293306218 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560f0bb7b680 con 0x560f0bb45400 2024-04-30T10:02:08.375 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:08 smithi046 bash[86005]: ce35c1eb8525dffd057b272472473105ce91bc83cf36d765bdfcf4da931ab120 2024-04-30T10:02:08.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:08 smithi026 ceph-mon[86310]: pgmap v231: 161 pgs: 9 active+undersized+degraded, 3 peering, 8 stale+active+clean, 17 active+undersized, 124 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 37/627 objects degraded (5.901%) 2024-04-30T10:02:08.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:08 smithi026 ceph-mon[80969]: pgmap v231: 161 pgs: 9 active+undersized+degraded, 3 peering, 8 stale+active+clean, 17 active+undersized, 124 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 37/627 objects degraded (5.901%) 2024-04-30T10:02:08.734 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:08 smithi046 systemd[1]: Started Ceph osd.6 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:02:09.234 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:02:08.816+0000 7fd88441b7c0 -1 Falling back to public interface 2024-04-30T10:02:09.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:09 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:09.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:09 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:09.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:09 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:09.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:09 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:09.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:09 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:09.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:09 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:09 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:09 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:09 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:10.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:10 smithi026 ceph-mon[86310]: pgmap v232: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-30T10:02:10.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:10 smithi026 ceph-mon[86310]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-30T10:02:10.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:10 smithi026 ceph-mon[80969]: pgmap v232: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-30T10:02:10.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:10 smithi026 ceph-mon[80969]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-30T10:02:10.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:10 smithi046 ceph-mon[65689]: pgmap v232: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-30T10:02:10.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:10 smithi046 ceph-mon[65689]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-30T10:02:11.879 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:02:11.563+0000 7fd88441b7c0 -1 osd.6 0 read_superblock omap replica is missing. 2024-04-30T10:02:11.879 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:02:11.591+0000 7fd88441b7c0 -1 osd.6 143 log_to_monitors true 2024-04-30T10:02:12.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:12 smithi026 ceph-mon[80969]: pgmap v233: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-30T10:02:12.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:12 smithi026 ceph-mon[80969]: from='osd.6 [v2:172.21.15.46:6816/3590550509,v1:172.21.15.46:6817/3590550509]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T10:02:12.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:12 smithi026 ceph-mon[80969]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T10:02:12.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:12 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 75/627 objects degraded (11.962%), 19 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:12.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:12 smithi026 ceph-mon[86310]: pgmap v233: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-30T10:02:12.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:12 smithi026 ceph-mon[86310]: from='osd.6 [v2:172.21.15.46:6816/3590550509,v1:172.21.15.46:6817/3590550509]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T10:02:12.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:12 smithi026 ceph-mon[86310]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T10:02:12.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:12 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 75/627 objects degraded (11.962%), 19 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:12 smithi046 ceph-mon[65689]: pgmap v233: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 210 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-30T10:02:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:12 smithi046 ceph-mon[65689]: from='osd.6 [v2:172.21.15.46:6816/3590550509,v1:172.21.15.46:6817/3590550509]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T10:02:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:12 smithi046 ceph-mon[65689]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-30T10:02:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:12 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 75/627 objects degraded (11.962%), 19 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:12.734 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:02:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:02:12.454+0000 7fd875872700 -1 osd.6 143 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T10:02:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:13 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:13 smithi046 ceph-mon[65689]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-30T10:02:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:13 smithi046 ceph-mon[65689]: from='osd.6 [v2:172.21.15.46:6816/3590550509,v1:172.21.15.46:6817/3590550509]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:13 smithi046 ceph-mon[65689]: osdmap e146: 8 total, 7 up, 8 in 2024-04-30T10:02:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:13 smithi046 ceph-mon[65689]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:13.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[80969]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[80969]: from='osd.6 [v2:172.21.15.46:6816/3590550509,v1:172.21.15.46:6817/3590550509]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[80969]: osdmap e146: 8 total, 7 up, 8 in 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[80969]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[86310]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[86310]: from='osd.6 [v2:172.21.15.46:6816/3590550509,v1:172.21.15.46:6817/3590550509]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:13.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[86310]: osdmap e146: 8 total, 7 up, 8 in 2024-04-30T10:02:13.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[86310]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:13.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:13.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:14.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:14 smithi046 ceph-mon[65689]: pgmap v235: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-30T10:02:14.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:14 smithi046 ceph-mon[65689]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:02:14.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:14 smithi046 ceph-mon[65689]: osd.6 [v2:172.21.15.46:6816/3590550509,v1:172.21.15.46:6817/3590550509] boot 2024-04-30T10:02:14.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:14 smithi046 ceph-mon[65689]: osdmap e147: 8 total, 8 up, 8 in 2024-04-30T10:02:14.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T10:02:14.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:14.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:14.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:14 smithi046 ceph-mon[65689]: osdmap e148: 8 total, 8 up, 8 in 2024-04-30T10:02:14.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[80969]: pgmap v235: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[80969]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[80969]: osd.6 [v2:172.21.15.46:6816/3590550509,v1:172.21.15.46:6817/3590550509] boot 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[80969]: osdmap e147: 8 total, 8 up, 8 in 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[80969]: osdmap e148: 8 total, 8 up, 8 in 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[86310]: pgmap v235: 161 pgs: 19 active+undersized+degraded, 34 active+undersized, 108 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 75/627 objects degraded (11.962%) 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[86310]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[86310]: osd.6 [v2:172.21.15.46:6816/3590550509,v1:172.21.15.46:6817/3590550509] boot 2024-04-30T10:02:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[86310]: osdmap e147: 8 total, 8 up, 8 in 2024-04-30T10:02:14.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-30T10:02:14.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:14.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:14.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:14 smithi026 ceph-mon[86310]: osdmap e148: 8 total, 8 up, 8 in 2024-04-30T10:02:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:02:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:02:16] "GET /metrics HTTP/1.1" 200 38067 "" "Prometheus/2.43.0" 2024-04-30T10:02:16.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:16 smithi026 ceph-mon[86310]: pgmap v238: 161 pgs: 1 peering, 17 active+undersized+degraded, 31 active+undersized, 112 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 65/627 objects degraded (10.367%) 2024-04-30T10:02:16.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:16.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:16.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:16 smithi026 ceph-mon[80969]: pgmap v238: 161 pgs: 1 peering, 17 active+undersized+degraded, 31 active+undersized, 112 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 65/627 objects degraded (10.367%) 2024-04-30T10:02:16.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:16.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:16.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:16 smithi046 ceph-mon[65689]: pgmap v238: 161 pgs: 1 peering, 17 active+undersized+degraded, 31 active+undersized, 112 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s; 65/627 objects degraded (10.367%) 2024-04-30T10:02:16.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:16.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:17.146 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:02:17.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:17 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 65/627 objects degraded (10.367%), 17 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:17.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:17 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 65/627 objects degraded (10.367%), 17 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:17.834 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:02:17.834 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (8m) 79s ago 30m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:02:17.834 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (8m) 4s ago 30m 81.7M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:02:17.834 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 79s ago 29m 62.3M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:02:17.834 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (9m) 4s ago 35m 455M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:02:17.834 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (19m) 79s ago 37m 508M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:02:17.834 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (5m) 79s ago 37m 53.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (6m) 4s ago 35m 45.2M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (5m) 79s ago 35m 44.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (9m) 79s ago 31m 16.0M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (9m) 4s ago 31m 16.7M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (3m) 79s ago 35m 48.3M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (3m) 79s ago 34m 44.4M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (2m) 79s ago 34m 41.5M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (84s) 79s ago 33m 15.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 8d72616b503d 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (64s) 4s ago 33m 51.1M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a 501e0c080dad 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (44s) 4s ago 33m 43.6M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b070c14531b0 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (9s) 4s ago 32m 12.7M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a ce35c1eb8525 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (32m) 4s ago 32m 61.0M 3249M 17.2.0 e1d6a67b021e 142143e5fc21 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (8m) 4s ago 31m 80.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (29m) 79s ago 29m 78.0M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T10:02:17.835 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (29m) 4s ago 29m 78.3M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T10:02:17.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:17 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 65/627 objects degraded (10.367%), 17 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:18.267 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:02:18.267 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T10:02:18.267 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 1, 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 7 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 3, 2024-04-30T10:02:18.268 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 12 2024-04-30T10:02:18.269 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T10:02:18.269 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:02:18.610 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: pgmap v239: 161 pgs: 1 peering, 10 active+undersized+degraded, 19 active+undersized, 131 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 38/627 objects degraded (6.061%) 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='client.54484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.612 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-30T10:02:18.612 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:18 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/1982295456' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.616 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:02:18.616 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T10:02:18.616 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T10:02:18.616 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons of type(s) crash,osd", 2024-04-30T10:02:18.616 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T10:02:18.617 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "7/8 daemons upgraded", 2024-04-30T10:02:18.617 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading osd daemons", 2024-04-30T10:02:18.617 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T10:02:18.617 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:02:18.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: pgmap v239: 161 pgs: 1 peering, 10 active+undersized+degraded, 19 active+undersized, 131 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 38/627 objects degraded (6.061%) 2024-04-30T10:02:18.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:18.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='client.54484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:18.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:18.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:18.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:18.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/1982295456' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: pgmap v239: 161 pgs: 1 peering, 10 active+undersized+degraded, 19 active+undersized, 131 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s; 38/627 objects degraded (6.061%) 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='client.54478 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='client.54484 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:18.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-30T10:02:18.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:18 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/1982295456' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:19 smithi046 ceph-mon[65689]: from='client.54490 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:19 smithi046 ceph-mon[65689]: pgmap v240: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-04-30T10:02:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:19 smithi046 ceph-mon[65689]: pgmap v241: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T10:02:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:19 smithi046 ceph-mon[65689]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-30T10:02:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:19 smithi046 ceph-mon[65689]: Upgrade: osd.7 is safe to restart 2024-04-30T10:02:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:19 smithi046 ceph-mon[65689]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/627 objects degraded (6.061%), 10 pgs degraded) 2024-04-30T10:02:19.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[86310]: from='client.54490 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:19.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[86310]: pgmap v240: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[86310]: pgmap v241: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[86310]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[86310]: Upgrade: osd.7 is safe to restart 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[86310]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/627 objects degraded (6.061%), 10 pgs degraded) 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[80969]: from='client.54490 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[80969]: pgmap v240: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 944 B/s rd, 0 op/s 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[80969]: pgmap v241: 161 pgs: 1 peering, 160 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[80969]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["7"], "max": 16}]: dispatch 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[80969]: Upgrade: osd.7 is safe to restart 2024-04-30T10:02:19.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:19 smithi026 ceph-mon[80969]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 38/627 objects degraded (6.061%), 10 pgs degraded) 2024-04-30T10:02:20.686 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:20 smithi046 ceph-mon[65689]: from='client.44568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:20.686 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:20.686 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-30T10:02:20.686 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:20.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:20 smithi026 ceph-mon[86310]: from='client.44568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:20.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:20.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-30T10:02:20.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:20.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:20 smithi026 ceph-mon[80969]: from='client.44568 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:20.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:20.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-30T10:02:20.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:21.641 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:21 smithi046 ceph-mon[65689]: Upgrade: Updating osd.7 2024-04-30T10:02:21.641 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:21 smithi046 ceph-mon[65689]: Deploying daemon osd.7 on smithi046 2024-04-30T10:02:21.641 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:21 smithi046 ceph-mon[65689]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 0 B/s wr, 3 op/s 2024-04-30T10:02:21.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:21 smithi026 ceph-mon[86310]: Upgrade: Updating osd.7 2024-04-30T10:02:21.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:21 smithi026 ceph-mon[86310]: Deploying daemon osd.7 on smithi046 2024-04-30T10:02:21.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:21 smithi026 ceph-mon[86310]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 0 B/s wr, 3 op/s 2024-04-30T10:02:21.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:21 smithi026 ceph-mon[80969]: Upgrade: Updating osd.7 2024-04-30T10:02:21.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:21 smithi026 ceph-mon[80969]: Deploying daemon osd.7 on smithi046 2024-04-30T10:02:21.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:21 smithi026 ceph-mon[80969]: pgmap v242: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 0 B/s wr, 3 op/s 2024-04-30T10:02:23.454 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:23 smithi046 systemd[1]: Stopping Ceph osd.7 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:02:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:23 smithi046 ceph-mon[65689]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 41 op/s 2024-04-30T10:02:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:23 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:23.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:23 smithi046 ceph-mon[65689]: osd.7 marked itself down and dead 2024-04-30T10:02:23.735 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[35380]: 2024-04-30T10:02:23.451+0000 7fb6ac061700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:02:23.735 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[35380]: 2024-04-30T10:02:23.451+0000 7fb6ac061700 -1 osd.7 148 *** Got signal Terminated *** 2024-04-30T10:02:23.735 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[35380]: 2024-04-30T10:02:23.451+0000 7fb6ac061700 -1 osd.7 148 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:02:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:23 smithi026 ceph-mon[80969]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 41 op/s 2024-04-30T10:02:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:23 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:23.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:23 smithi026 ceph-mon[80969]: osd.7 marked itself down and dead 2024-04-30T10:02:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:23 smithi026 ceph-mon[86310]: pgmap v243: 161 pgs: 161 active+clean; 585 KiB data, 226 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 0 B/s wr, 41 op/s 2024-04-30T10:02:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:23 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:23.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:23 smithi026 ceph-mon[86310]: osd.7 marked itself down and dead 2024-04-30T10:02:24.071 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:23 smithi046 podman[90554]: 2024-04-30 10:02:23.811456619 +0000 UTC m=+0.464122894 container died 142143e5fc21795cfe39bdb1facd0eaecd7e00ac7091abf9d227701618d9a9f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, vcs-type=git, com.redhat.component=centos-stream-container, vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git, architecture=x86_64, ceph=True, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, version=8, release=754, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.expose-services=, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, build-date=2022-05-03T08:36:31.336870, distribution-scope=public, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-30T10:02:24.379 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:24 smithi046 podman[90554]: 2024-04-30 10:02:24.160503661 +0000 UTC m=+0.813169937 container cleanup 142143e5fc21795cfe39bdb1facd0eaecd7e00ac7091abf9d227701618d9a9f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7, build-date=2022-05-03T08:36:31.336870, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, ceph=True, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, GIT_BRANCH=HEAD, GIT_CLEAN=True, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, maintainer=Guillaume Abrioux , RELEASE=HEAD, vendor=Red Hat, Inc., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, CEPH_POINT_RELEASE=-17.2.0, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, release=754, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-30T10:02:24.379 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:24 smithi046 bash[90554]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7 2024-04-30T10:02:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:24 smithi046 ceph-mon[65689]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:02:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:24 smithi046 ceph-mon[65689]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-30T10:02:24.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:24 smithi046 ceph-mon[65689]: osdmap e149: 8 total, 7 up, 8 in 2024-04-30T10:02:24.734 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:24 smithi046 podman[90572]: 2024-04-30 10:02:24.378800355 +0000 UTC m=+0.560315514 container remove 142143e5fc21795cfe39bdb1facd0eaecd7e00ac7091abf9d227701618d9a9f0 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.openshift.tags=base centos centos-stream, distribution-scope=public, CEPH_POINT_RELEASE=-17.2.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_CLEAN=True, com.redhat.component=centos-stream-container, io.k8s.display-name=CentOS Stream 8, version=8, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, release=754, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, name=centos-stream, io.buildah.version=1.19.8, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vcs-type=git, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, architecture=x86_64, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com) 2024-04-30T10:02:24.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:24 smithi026 ceph-mon[80969]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:02:24.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:24 smithi026 ceph-mon[80969]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-30T10:02:24.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:24 smithi026 ceph-mon[80969]: osdmap e149: 8 total, 7 up, 8 in 2024-04-30T10:02:24.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:24 smithi026 ceph-mon[86310]: Health check failed: 1 osds down (OSD_DOWN) 2024-04-30T10:02:24.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:24 smithi026 ceph-mon[86310]: Health check failed: all OSDs are running squid or later but require_osd_release < squid (OSD_UPGRADE_FINISHED) 2024-04-30T10:02:24.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:24 smithi026 ceph-mon[86310]: osdmap e149: 8 total, 7 up, 8 in 2024-04-30T10:02:25.116 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:24 smithi046 podman[90664]: 2024-04-30 10:02:24.846513478 +0000 UTC m=+0.110239182 container create 785878a0cd2c4f1d6995f6e39db83cf6d614936f064951ca524b95896c12b883 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-04-30T10:02:25.116 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:24 smithi046 podman[90664]: 2024-04-30 10:02:24.755231707 +0000 UTC m=+0.018957367 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:02:25.116 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:25 smithi046 podman[90664]: 2024-04-30 10:02:25.040464901 +0000 UTC m=+0.304190561 container init 785878a0cd2c4f1d6995f6e39db83cf6d614936f064951ca524b95896c12b883 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=) 2024-04-30T10:02:25.116 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:25 smithi046 podman[90664]: 2024-04-30 10:02:25.04412661 +0000 UTC m=+0.307852276 container start 785878a0cd2c4f1d6995f6e39db83cf6d614936f064951ca524b95896c12b883 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-04-30T10:02:25.116 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:25 smithi046 podman[90664]: 2024-04-30 10:02:25.115387944 +0000 UTC m=+0.379113615 container attach 785878a0cd2c4f1d6995f6e39db83cf6d614936f064951ca524b95896c12b883 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:02:25.472 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:25 smithi046 podman[90664]: 2024-04-30 10:02:25.354092911 +0000 UTC m=+0.617818571 container died 785878a0cd2c4f1d6995f6e39db83cf6d614936f064951ca524b95896c12b883 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-04-30T10:02:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:25 smithi046 ceph-mon[65689]: pgmap v245: 161 pgs: 13 peering, 20 stale+active+clean, 128 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 0 B/s wr, 135 op/s 2024-04-30T10:02:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:25 smithi046 ceph-mon[65689]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-30T10:02:25.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:25 smithi046 ceph-mon[65689]: osdmap e150: 8 total, 7 up, 8 in 2024-04-30T10:02:25.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:25 smithi026 ceph-mon[80969]: pgmap v245: 161 pgs: 13 peering, 20 stale+active+clean, 128 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 0 B/s wr, 135 op/s 2024-04-30T10:02:25.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:25 smithi026 ceph-mon[80969]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-30T10:02:25.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:25 smithi026 ceph-mon[80969]: osdmap e150: 8 total, 7 up, 8 in 2024-04-30T10:02:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:25 smithi026 ceph-mon[86310]: pgmap v245: 161 pgs: 13 peering, 20 stale+active+clean, 128 active+clean; 585 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 0 B/s wr, 135 op/s 2024-04-30T10:02:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:25 smithi026 ceph-mon[86310]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-04-30T10:02:25.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:25 smithi026 ceph-mon[86310]: osdmap e150: 8 total, 7 up, 8 in 2024-04-30T10:02:26.146 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:25 smithi046 podman[90690]: 2024-04-30 10:02:25.910626179 +0000 UTC m=+0.551304987 container remove 785878a0cd2c4f1d6995f6e39db83cf6d614936f064951ca524b95896c12b883 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-04-30T10:02:26.146 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:25 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.7.service: Deactivated successfully. 2024-04-30T10:02:26.146 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:25 smithi046 systemd[1]: Stopped Ceph osd.7 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:02:26.146 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:25 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.7.service: Consumed 19.319s CPU time. 2024-04-30T10:02:26.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:26 smithi046 systemd[1]: Starting Ceph osd.7 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:02:26.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:26 smithi046 podman[90797]: 2024-04-30 10:02:26.270420854 +0000 UTC m=+0.018808462 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:02:26.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:26 smithi046 podman[90797]: 2024-04-30 10:02:26.387159307 +0000 UTC m=+0.135546891 container create 842cbb370329f3ff7fcaa28c99b0e18d94c08db8186c47bac017ef3e2af23403 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:02:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:02:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:02:26] "GET /metrics HTTP/1.1" 200 38067 "" "Prometheus/2.43.0" 2024-04-30T10:02:26.984 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:26 smithi046 podman[90797]: 2024-04-30 10:02:26.606188979 +0000 UTC m=+0.354576561 container init 842cbb370329f3ff7fcaa28c99b0e18d94c08db8186c47bac017ef3e2af23403 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-30T10:02:26.984 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:26 smithi046 podman[90797]: 2024-04-30 10:02:26.609851484 +0000 UTC m=+0.358239066 container start 842cbb370329f3ff7fcaa28c99b0e18d94c08db8186c47bac017ef3e2af23403 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:02:26.984 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:26 smithi046 podman[90797]: 2024-04-30 10:02:26.71465411 +0000 UTC m=+0.463041702 container attach 842cbb370329f3ff7fcaa28c99b0e18d94c08db8186c47bac017ef3e2af23403 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-30T10:02:27.410 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-mon[65689]: pgmap v247: 161 pgs: 3 active+undersized+degraded, 13 peering, 19 stale+active+clean, 2 active+undersized, 124 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 0 B/s wr, 138 op/s; 4/627 objects degraded (0.638%) 2024-04-30T10:02:27.410 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:27.410 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-mon[65689]: Health check failed: Degraded data redundancy: 4/627 objects degraded (0.638%), 3 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:27.410 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate[90812]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:02:27.410 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate[90812]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:02:27.410 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 bash[90797]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 bash[90797]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 bash[90797]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 bash[90797]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 bash[90797]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 bash[90797]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 bash[90797]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 bash[90797]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 bash[90797]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate[90812]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate[90812]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-7 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate[90812]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate[90812]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate[90812]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-7/block 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate[90812]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2024-04-30T10:02:27.411 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate[90812]: --> ceph-volume raw activate successful for osd ID: 7 2024-04-30T10:02:27.412 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 podman[90797]: 2024-04-30 10:02:27.12552338 +0000 UTC m=+0.873910972 container died 842cbb370329f3ff7fcaa28c99b0e18d94c08db8186c47bac017ef3e2af23403 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5) 2024-04-30T10:02:27.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:27 smithi026 ceph-mon[80969]: pgmap v247: 161 pgs: 3 active+undersized+degraded, 13 peering, 19 stale+active+clean, 2 active+undersized, 124 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 0 B/s wr, 138 op/s; 4/627 objects degraded (0.638%) 2024-04-30T10:02:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:27 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:27.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:27 smithi026 ceph-mon[80969]: Health check failed: Degraded data redundancy: 4/627 objects degraded (0.638%), 3 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:27 smithi026 ceph-mon[86310]: pgmap v247: 161 pgs: 3 active+undersized+degraded, 13 peering, 19 stale+active+clean, 2 active+undersized, 124 active+clean; 585 KiB data, 234 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 0 B/s wr, 138 op/s; 4/627 objects degraded (0.638%) 2024-04-30T10:02:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:27 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:27.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:27 smithi026 ceph-mon[86310]: Health check failed: Degraded data redundancy: 4/627 objects degraded (0.638%), 3 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:27.984 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 podman[90934]: 2024-04-30 10:02:27.71866689 +0000 UTC m=+0.587808846 container remove 842cbb370329f3ff7fcaa28c99b0e18d94c08db8186c47bac017ef3e2af23403 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-activate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-30T10:02:27.984 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 podman[90994]: 2024-04-30 10:02:27.844639052 +0000 UTC m=+0.018947671 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:02:27.984 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:27 smithi046 podman[90994]: 2024-04-30 10:02:27.944849433 +0000 UTC m=+0.119158097 container create b081d959354ecd93f8e37573a90b05969960cfbfdd70e22065790e5fd9eaf319 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-04-30T10:02:28.319 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:28 smithi046 podman[90994]: 2024-04-30 10:02:28.138584273 +0000 UTC m=+0.312892895 container init b081d959354ecd93f8e37573a90b05969960cfbfdd70e22065790e5fd9eaf319 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:02:28.319 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:28 smithi046 podman[90994]: 2024-04-30 10:02:28.141624748 +0000 UTC m=+0.315933367 container start b081d959354ecd93f8e37573a90b05969960cfbfdd70e22065790e5fd9eaf319 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-30T10:02:28.319 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:28 smithi046 ceph-osd[91012]: -- 172.21.15.46:0/3272651698 <== mon.2 v2:172.21.15.46:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5647b7249680 con 0x5647b7212c00 2024-04-30T10:02:28.319 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:28 smithi046 bash[90994]: b081d959354ecd93f8e37573a90b05969960cfbfdd70e22065790e5fd9eaf319 2024-04-30T10:02:28.319 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:28 smithi046 systemd[1]: Started Ceph osd.7 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:02:28.984 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:28 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:02:28.692+0000 7f759a92a7c0 -1 Falling back to public interface 2024-04-30T10:02:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:29 smithi046 ceph-mon[65689]: pgmap v248: 161 pgs: 23 active+undersized+degraded, 13 peering, 33 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 0 B/s wr, 174 op/s; 75/627 objects degraded (11.962%) 2024-04-30T10:02:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:29 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:29 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:29.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:29 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:29.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:29 smithi026 ceph-mon[86310]: pgmap v248: 161 pgs: 23 active+undersized+degraded, 13 peering, 33 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 0 B/s wr, 174 op/s; 75/627 objects degraded (11.962%) 2024-04-30T10:02:29.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:29 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:29 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:29.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:29 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:29.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:29 smithi026 ceph-mon[80969]: pgmap v248: 161 pgs: 23 active+undersized+degraded, 13 peering, 33 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 0 B/s wr, 174 op/s; 75/627 objects degraded (11.962%) 2024-04-30T10:02:29.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:29 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:29.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:29 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:29.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:29 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:30.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:30 smithi026 ceph-mon[86310]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-04-30T10:02:30.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:30 smithi026 ceph-mon[80969]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-04-30T10:02:30.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:30 smithi046 ceph-mon[65689]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-04-30T10:02:31.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:31 smithi026 ceph-mon[80969]: pgmap v249: 161 pgs: 29 active+undersized+degraded, 40 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 0 B/s wr, 139 op/s; 87/627 objects degraded (13.876%) 2024-04-30T10:02:31.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:31.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:31.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:31 smithi026 ceph-mon[86310]: pgmap v249: 161 pgs: 29 active+undersized+degraded, 40 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 0 B/s wr, 139 op/s; 87/627 objects degraded (13.876%) 2024-04-30T10:02:31.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:31.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:31.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:31 smithi046 ceph-mon[65689]: pgmap v249: 161 pgs: 29 active+undersized+degraded, 40 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 0 B/s wr, 139 op/s; 87/627 objects degraded (13.876%) 2024-04-30T10:02:31.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:31.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:31.734 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:02:31.430+0000 7f759a92a7c0 -1 osd.7 0 read_superblock omap replica is missing. 2024-04-30T10:02:31.734 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:31 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:02:31.461+0000 7f759a92a7c0 -1 osd.7 148 log_to_monitors true 2024-04-30T10:02:32.666 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:32 smithi046 ceph-mon[65689]: from='osd.7 [v2:172.21.15.46:6824/3028328451,v1:172.21.15.46:6825/3028328451]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T10:02:32.666 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:32 smithi046 ceph-mon[65689]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T10:02:32.666 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:32 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 87/627 objects degraded (13.876%), 29 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:32.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:32 smithi026 ceph-mon[86310]: from='osd.7 [v2:172.21.15.46:6824/3028328451,v1:172.21.15.46:6825/3028328451]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T10:02:32.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:32 smithi026 ceph-mon[86310]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T10:02:32.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:32 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 87/627 objects degraded (13.876%), 29 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:32.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:32 smithi026 ceph-mon[80969]: from='osd.7 [v2:172.21.15.46:6824/3028328451,v1:172.21.15.46:6825/3028328451]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T10:02:32.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:32 smithi026 ceph-mon[80969]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-30T10:02:32.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:32 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 87/627 objects degraded (13.876%), 29 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:32.984 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:02:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:02:32.665+0000 7f7587d79700 -1 osd.7 148 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-30T10:02:33.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[80969]: pgmap v250: 161 pgs: 29 active+undersized+degraded, 40 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 84 op/s; 87/627 objects degraded (13.876%) 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[80969]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[80969]: osdmap e151: 8 total, 7 up, 8 in 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[80969]: from='osd.7 [v2:172.21.15.46:6824/3028328451,v1:172.21.15.46:6825/3028328451]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[80969]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[80969]: from='osd.7 ' entity='osd.7' 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[86310]: pgmap v250: 161 pgs: 29 active+undersized+degraded, 40 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 84 op/s; 87/627 objects degraded (13.876%) 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:33.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[86310]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-30T10:02:33.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[86310]: osdmap e151: 8 total, 7 up, 8 in 2024-04-30T10:02:33.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[86310]: from='osd.7 [v2:172.21.15.46:6824/3028328451,v1:172.21.15.46:6825/3028328451]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:33.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[86310]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:33.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[86310]: from='osd.7 ' entity='osd.7' 2024-04-30T10:02:33.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:33.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:33 smithi046 ceph-mon[65689]: pgmap v250: 161 pgs: 29 active+undersized+degraded, 40 active+undersized, 92 active+clean; 585 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 50 KiB/s rd, 0 B/s wr, 84 op/s; 87/627 objects degraded (13.876%) 2024-04-30T10:02:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:33 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:33 smithi046 ceph-mon[65689]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-30T10:02:33.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:33 smithi046 ceph-mon[65689]: osdmap e151: 8 total, 7 up, 8 in 2024-04-30T10:02:33.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:33 smithi046 ceph-mon[65689]: from='osd.7 [v2:172.21.15.46:6824/3028328451,v1:172.21.15.46:6825/3028328451]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:33.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:33 smithi046 ceph-mon[65689]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi046", "root=default"]}]: dispatch 2024-04-30T10:02:33.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:33 smithi046 ceph-mon[65689]: from='osd.7 ' entity='osd.7' 2024-04-30T10:02:33.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:33.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:34.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:34 smithi046 ceph-mon[65689]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:02:34.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:34 smithi046 ceph-mon[65689]: osd.7 [v2:172.21.15.46:6824/3028328451,v1:172.21.15.46:6825/3028328451] boot 2024-04-30T10:02:34.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:34 smithi046 ceph-mon[65689]: osdmap e152: 8 total, 8 up, 8 in 2024-04-30T10:02:34.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:34 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T10:02:34.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:34 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:34.645 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:34 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:34.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[86310]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[86310]: osd.7 [v2:172.21.15.46:6824/3028328451,v1:172.21.15.46:6825/3028328451] boot 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[86310]: osdmap e152: 8 total, 8 up, 8 in 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[80969]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[80969]: osd.7 [v2:172.21.15.46:6824/3028328451,v1:172.21.15.46:6825/3028328451] boot 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[80969]: osdmap e152: 8 total, 8 up, 8 in 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:34.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:34 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:35.663 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:35 smithi046 ceph-mon[65689]: pgmap v253: 161 pgs: 10 peering, 25 active+undersized+degraded, 34 active+undersized, 92 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 39 op/s; 78/627 objects degraded (12.440%) 2024-04-30T10:02:35.663 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:35 smithi046 ceph-mon[65689]: osdmap e153: 8 total, 8 up, 8 in 2024-04-30T10:02:35.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:35 smithi026 ceph-mon[86310]: pgmap v253: 161 pgs: 10 peering, 25 active+undersized+degraded, 34 active+undersized, 92 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 39 op/s; 78/627 objects degraded (12.440%) 2024-04-30T10:02:35.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:35 smithi026 ceph-mon[86310]: osdmap e153: 8 total, 8 up, 8 in 2024-04-30T10:02:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:35 smithi026 ceph-mon[80969]: pgmap v253: 161 pgs: 10 peering, 25 active+undersized+degraded, 34 active+undersized, 92 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 0 B/s wr, 39 op/s; 78/627 objects degraded (12.440%) 2024-04-30T10:02:35.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:35 smithi026 ceph-mon[80969]: osdmap e153: 8 total, 8 up, 8 in 2024-04-30T10:02:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:02:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:02:36] "GET /metrics HTTP/1.1" 200 38294 "" "Prometheus/2.43.0" 2024-04-30T10:02:37.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:37 smithi026 ceph-mon[86310]: pgmap v255: 161 pgs: 10 peering, 22 active+undersized+degraded, 33 active+undersized, 96 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 74/627 objects degraded (11.802%) 2024-04-30T10:02:37.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:37 smithi026 ceph-mon[86310]: Health check update: Degraded data redundancy: 74/627 objects degraded (11.802%), 22 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:37.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:37 smithi026 ceph-mon[80969]: pgmap v255: 161 pgs: 10 peering, 22 active+undersized+degraded, 33 active+undersized, 96 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 74/627 objects degraded (11.802%) 2024-04-30T10:02:37.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:37 smithi026 ceph-mon[80969]: Health check update: Degraded data redundancy: 74/627 objects degraded (11.802%), 22 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:37.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:37 smithi046 ceph-mon[65689]: pgmap v255: 161 pgs: 10 peering, 22 active+undersized+degraded, 33 active+undersized, 96 active+clean; 585 KiB data, 249 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s; 74/627 objects degraded (11.802%) 2024-04-30T10:02:37.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:37 smithi046 ceph-mon[65689]: Health check update: Degraded data redundancy: 74/627 objects degraded (11.802%), 22 pgs degraded (PG_DEGRADED) 2024-04-30T10:02:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: pgmap v256: 161 pgs: 10 peering, 6 active+undersized+degraded, 6 active+undersized, 139 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s; 10/627 objects degraded (1.595%) 2024-04-30T10:02:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: pgmap v257: 161 pgs: 10 peering, 6 active+undersized+degraded, 6 active+undersized, 139 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s; 10/627 objects degraded (1.595%) 2024-04-30T10:02:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:38.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all osd 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-30T10:02:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: pgmap v256: 161 pgs: 10 peering, 6 active+undersized+degraded, 6 active+undersized, 139 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s; 10/627 objects degraded (1.595%) 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: pgmap v257: 161 pgs: 10 peering, 6 active+undersized+degraded, 6 active+undersized, 139 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s; 10/627 objects degraded (1.595%) 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all osd 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-30T10:02:38.700 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-30T10:02:38.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:38 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-30T10:02:38.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:38.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:38.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: pgmap v256: 161 pgs: 10 peering, 6 active+undersized+degraded, 6 active+undersized, 139 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 917 B/s rd, 0 op/s; 10/627 objects degraded (1.595%) 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: pgmap v257: 161 pgs: 10 peering, 6 active+undersized+degraded, 6 active+undersized, 139 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1013 B/s rd, 0 op/s; 10/627 objects degraded (1.595%) 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all osd 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]: dispatch 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.0"}]': finished 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]: dispatch 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.1"}]': finished 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]: dispatch 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.2"}]': finished 2024-04-30T10:02:38.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]: dispatch 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.3"}]': finished 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]: dispatch 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.4"}]': finished 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]: dispatch 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.5"}]': finished 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]: dispatch 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.6"}]': finished 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]: dispatch 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd.7"}]': finished 2024-04-30T10:02:38.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:38 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd require-osd-release", "release": "squid"}]: dispatch 2024-04-30T10:02:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: Upgrade: Setting require_osd_release to 19 squid 2024-04-30T10:02:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-30T10:02:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-30T10:02:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: osdmap e154: 8 total, 8 up, 8 in 2024-04-30T10:02:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: Upgrade: Setting require_osd_release to 19 squid 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-30T10:02:39.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: osdmap e154: 8 total, 8 up, 8 in 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.700 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T10:02:39.701 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T10:02:39.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T10:02:39.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T10:02:39.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:39.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:39 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: Upgrade: Setting require_osd_release to 19 squid 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: Health check cleared: OSD_UPGRADE_FINISHED (was: all OSDs are running squid or later but require_osd_release < squid) 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "osd require-osd-release", "release": "squid"}]': finished 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: osdmap e154: 8 total, 8 up, 8 in 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T10:02:39.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.737 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:02:39.738 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:39 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:40.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mds 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all nfs 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all nvmeof 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: Upgrade: Finalizing container_image settings 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: Upgrade: Complete! 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: pgmap v259: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: pgmap v260: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-30T10:02:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:02:40.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[86310]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/627 objects degraded (1.595%), 6 pgs degraded) 2024-04-30T10:02:40.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mds 2024-04-30T10:02:40.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T10:02:40.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T10:02:40.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all nfs 2024-04-30T10:02:40.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all nvmeof 2024-04-30T10:02:40.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: Upgrade: Finalizing container_image settings 2024-04-30T10:02:40.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: Upgrade: Complete! 2024-04-30T10:02:40.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: pgmap v259: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T10:02:40.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: pgmap v260: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:40.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-30T10:02:40.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:02:40.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:40 smithi026 ceph-mon[80969]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/627 objects degraded (1.595%), 6 pgs degraded) 2024-04-30T10:02:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mds 2024-04-30T10:02:40.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T10:02:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T10:02:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nfs 2024-04-30T10:02:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nvmeof 2024-04-30T10:02:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: Upgrade: Finalizing container_image settings 2024-04-30T10:02:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: Upgrade: Complete! 2024-04-30T10:02:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: pgmap v259: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T10:02:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: pgmap v260: 161 pgs: 10 peering, 151 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:40.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: pgmap v261: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 1 op/s 2024-04-30T10:02:40.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: pgmap v262: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:02:40.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:40 smithi046 ceph-mon[65689]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/627 objects degraded (1.595%), 6 pgs degraded) 2024-04-30T10:02:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:42 smithi026 ceph-mon[86310]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:02:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:42 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:42.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:42 smithi026 ceph-mon[80969]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:02:42.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:42 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:42.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:42 smithi046 ceph-mon[65689]: pgmap v263: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:02:42.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:42 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:43.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:43 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:43.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:43 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:43 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:44.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:44 smithi026 ceph-mon[80969]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:44 smithi026 ceph-mon[86310]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:44 smithi046 ceph-mon[65689]: pgmap v264: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:46.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:46 smithi026 ceph-mon[80969]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-04-30T10:02:46.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:46.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:46.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:02:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:02:46] "GET /metrics HTTP/1.1" 200 38287 "" "Prometheus/2.43.0" 2024-04-30T10:02:46.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:46 smithi026 ceph-mon[86310]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-04-30T10:02:46.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:46.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:46 smithi046 ceph-mon[65689]: pgmap v265: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 832 B/s rd, 0 op/s 2024-04-30T10:02:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:02:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:47 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:47.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:47 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:47.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:47 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:02:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:48 smithi046 ceph-mon[65689]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:48.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:48 smithi026 ceph-mon[80969]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:48.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:48 smithi026 ceph-mon[86310]: pgmap v266: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:49.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T10:02:50.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:50 smithi026 ceph-mon[86310]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:02:50.322 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:50 smithi026 ceph-mon[86310]: from='client.54511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:50.322 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:50 smithi026 ceph-mon[80969]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:02:50.322 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:50 smithi026 ceph-mon[80969]: from='client.54511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:50 smithi046 ceph-mon[65689]: pgmap v267: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:02:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:50 smithi046 ceph-mon[65689]: from='client.54511 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:51.908 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (8m) 113s ago 31m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (8m) 19s ago 31m 82.2M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 113s ago 30m 62.3M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (10m) 19s ago 35m 455M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (20m) 113s ago 37m 508M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (6m) 113s ago 37m 53.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (7m) 19s ago 36m 46.3M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (6m) 113s ago 36m 44.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (9m) 113s ago 32m 16.0M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (9m) 19s ago 31m 17.1M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (4m) 113s ago 35m 48.3M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (4m) 113s ago 35m 44.4M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (2m) 113s ago 34m 41.5M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:02:51.909 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (118s) 113s ago 34m 15.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 8d72616b503d 2024-04-30T10:02:51.910 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (98s) 19s ago 34m 54.0M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a 501e0c080dad 2024-04-30T10:02:51.910 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (78s) 19s ago 33m 45.7M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b070c14531b0 2024-04-30T10:02:51.910 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (43s) 19s ago 33m 44.2M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a ce35c1eb8525 2024-04-30T10:02:51.910 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (24s) 19s ago 32m 12.5M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b081d959354e 2024-04-30T10:02:51.910 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (9m) 19s ago 32m 80.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:02:51.910 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (30m) 113s ago 30m 78.0M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T10:02:51.910 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (30m) 19s ago 30m 78.8M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T10:02:52.275 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:52 smithi026 ceph-mon[86310]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:52.275 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:52 smithi026 ceph-mon[80969]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:52.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:52 smithi046 ceph-mon[65689]: pgmap v268: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:52.694 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.osd | length == 1'"'"'' 2024-04-30T10:02:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:53 smithi046 ceph-mon[65689]: from='client.44583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:53 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:53.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:53 smithi026 ceph-mon[86310]: from='client.44583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:53.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:53 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:53.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:53 smithi026 ceph-mon[80969]: from='client.44583 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:02:53.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:53 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:02:54.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:54 smithi026 ceph-mon[86310]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:54.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:54 smithi026 ceph-mon[80969]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:54 smithi046 ceph-mon[65689]: pgmap v269: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:54.838 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:02:55.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:55 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2470826525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:55.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:55 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2470826525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:55 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2470826525' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:55.577 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.osd | keys'"'"' | grep $sha1' 2024-04-30T10:02:56.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:56 smithi046 ceph-mon[65689]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:56.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:56 smithi026 ceph-mon[86310]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:56.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:56 smithi026 ceph-mon[80969]: pgmap v270: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:02:56.582 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:02:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:02:56] "GET /metrics HTTP/1.1" 200 38287 "" "Prometheus/2.43.0" 2024-04-30T10:02:57.772 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)" 2024-04-30T10:02:58.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:58 smithi026 ceph-mon[86310]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:58.399 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:02:58 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2429609765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:58.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:58 smithi026 ceph-mon[80969]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:58.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:02:58 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2429609765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:02:58.427 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade status' 2024-04-30T10:02:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:58 smithi046 ceph-mon[65689]: pgmap v271: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:02:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:02:58 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2429609765' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:00.387 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:03:00.387 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-30T10:03:00.388 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-30T10:03:00.388 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-30T10:03:00.388 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T10:03:00.388 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-30T10:03:00.388 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-30T10:03:00.388 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T10:03:00.388 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:03:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:00 smithi046 ceph-mon[65689]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:00.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:00 smithi026 ceph-mon[86310]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:00.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:00 smithi026 ceph-mon[80969]: pgmap v272: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:01.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T10:03:01.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:01 smithi026 ceph-mon[86310]: from='client.54535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:01.354 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:01.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:01 smithi026 ceph-mon[80969]: from='client.54535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:01.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:01 smithi046 ceph-mon[65689]: from='client.54535 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:02.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:02 smithi026 ceph-mon[86310]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:02.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:02 smithi026 ceph-mon[80969]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:02 smithi046 ceph-mon[65689]: pgmap v273: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:03.219 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:03:03.220 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:03:03.220 INFO:teuthology.orchestra.run.smithi026.stdout: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T10:03:03.537 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:03 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:03.537 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:03 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/1908972409' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:03:03.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:03 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:03.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:03 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/1908972409' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:03:03.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:03 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:03.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:03 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/1908972409' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:03:03.830 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo' 2024-04-30T10:03:04.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:04 smithi026 ceph-mon[86310]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:04.436 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:04 smithi026 ceph-mon[80969]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:04.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:04 smithi046 ceph-mon[65689]: pgmap v274: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:06.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:06 smithi026 ceph-mon[86310]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:06.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:06 smithi026 ceph-mon[80969]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:06.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:03:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:03:06] "GET /metrics HTTP/1.1" 200 38299 "" "Prometheus/2.43.0" 2024-04-30T10:03:06.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:06 smithi046 ceph-mon[65689]: pgmap v275: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:07.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:07 smithi026 ceph-mon[86310]: from='client.44613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:07.396 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:07 smithi026 ceph-mon[80969]: from='client.44613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:07.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:07 smithi046 ceph-mon[65689]: from='client.44613 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "services": "rgw.foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:08.328 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:03:08.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:08 smithi026 ceph-mon[86310]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:08.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:08 smithi026 ceph-mon[80969]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:08.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:08 smithi046 ceph-mon[65689]: pgmap v276: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:09.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-04-30T10:03:09.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[86310]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:03:09.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:09.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[86310]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 892 B/s rd, 0 op/s 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[86310]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[86310]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[80969]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:09.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[80969]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 892 B/s rd, 0 op/s 2024-04-30T10:03:09.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[80969]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:09.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:09.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:09 smithi026 ceph-mon[80969]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:03:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:09 smithi046 ceph-mon[65689]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:03:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:09 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:09 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:09 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:09 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:09.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:09 smithi046 ceph-mon[65689]: pgmap v277: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 892 B/s rd, 0 op/s 2024-04-30T10:03:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:09 smithi046 ceph-mon[65689]: pgmap v278: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:09 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:09.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:09 smithi046 ceph-mon[65689]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:03:11.619 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:03:11.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:11 smithi026 ceph-mon[80969]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:11.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:11 smithi026 ceph-mon[86310]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:11.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:11 smithi046 ceph-mon[65689]: pgmap v279: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:12.334 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (9m) 2m ago 31m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (8m) 39s ago 31m 82.2M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 2m ago 30m 62.3M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (10m) 39s ago 36m 455M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (20m) 2m ago 38m 508M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (6m) 2m ago 38m 53.5M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (7m) 39s ago 36m 46.3M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (6m) 2m ago 36m 44.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (10m) 2m ago 32m 16.0M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (10m) 39s ago 32m 17.1M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (4m) 2m ago 35m 48.3M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (4m) 2m ago 35m 44.4M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:03:12.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (3m) 2m ago 35m 41.5M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:03:12.336 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (2m) 2m ago 34m 15.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 8d72616b503d 2024-04-30T10:03:12.336 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (118s) 39s ago 34m 54.0M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a 501e0c080dad 2024-04-30T10:03:12.336 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (99s) 39s ago 33m 45.7M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b070c14531b0 2024-04-30T10:03:12.336 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (64s) 39s ago 33m 44.2M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a ce35c1eb8525 2024-04-30T10:03:12.336 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (44s) 39s ago 33m 12.5M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b081d959354e 2024-04-30T10:03:12.336 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (9m) 39s ago 32m 80.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:03:12.336 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (30m) 2m ago 30m 78.0M - 17.2.0 e1d6a67b021e db894327195b 2024-04-30T10:03:12.336 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (30m) 39s ago 30m 78.8M - 17.2.0 e1d6a67b021e 0ba20930756d 2024-04-30T10:03:12.783 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:03:12.783 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T10:03:12.783 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T10:03:12.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:03:12.783 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 8 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)": 2, 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 13 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T10:03:12.784 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:03:13.146 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:03:13.146 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T10:03:13.146 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T10:03:13.146 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading daemons in service(s) rgw.foo", 2024-04-30T10:03:13.146 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T10:03:13.146 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "0/2 daemons upgraded", 2024-04-30T10:03:13.146 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading rgw daemons", 2024-04-30T10:03:13.146 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T10:03:13.146 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='client.54553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='client.44625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mgr 2024-04-30T10:03:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mon 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all crash 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all osd 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mds 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='client.54565 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:13.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/1017564741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='client.54553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='client.44625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mgr 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mon 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all crash 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all osd 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mds 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='client.54565 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:13.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:13 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/1017564741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='client.54553 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='client.44625 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mgr 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mon 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all crash 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all osd 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mds 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='client.54565 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: pgmap v280: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:13.736 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:13 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/1017564741' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:14.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:14 smithi026 ceph-mon[86310]: from='client.44643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:14.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:14.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T10:03:14.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:14 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:14.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:14 smithi026 ceph-mon[80969]: from='client.44643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:14.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:14.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T10:03:14.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:14 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:14.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:14 smithi046 ceph-mon[65689]: from='client.44643 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:14.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:14.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi026.igccwv", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T10:03:14.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:14 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:15.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:15 smithi026 ceph-mon[80969]: Upgrade: Updating rgw.foo.smithi026.igccwv (1/2) 2024-04-30T10:03:15.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:15 smithi026 ceph-mon[80969]: Deploying daemon rgw.foo.smithi026.igccwv on smithi026 2024-04-30T10:03:15.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:15 smithi026 ceph-mon[80969]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:15.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:15 smithi026 ceph-mon[86310]: Upgrade: Updating rgw.foo.smithi026.igccwv (1/2) 2024-04-30T10:03:15.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:15 smithi026 ceph-mon[86310]: Deploying daemon rgw.foo.smithi026.igccwv on smithi026 2024-04-30T10:03:15.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:15 smithi026 ceph-mon[86310]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:15.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:15 smithi046 ceph-mon[65689]: Upgrade: Updating rgw.foo.smithi026.igccwv (1/2) 2024-04-30T10:03:15.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:15 smithi046 ceph-mon[65689]: Deploying daemon rgw.foo.smithi026.igccwv on smithi026 2024-04-30T10:03:15.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:15 smithi046 ceph-mon[65689]: pgmap v281: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:03:16.635 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:16.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:16.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:16.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:03:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:03:16] "GET /metrics HTTP/1.1" 200 38294 "" "Prometheus/2.43.0" 2024-04-30T10:03:17.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:17 smithi026 ceph-mon[86310]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 540 B/s rd, 0 op/s 2024-04-30T10:03:17.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:17 smithi026 ceph-mon[80969]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 540 B/s rd, 0 op/s 2024-04-30T10:03:17.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:17 smithi046 ceph-mon[65689]: pgmap v282: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 540 B/s rd, 0 op/s 2024-04-30T10:03:19.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:19 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:19.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:19 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:19.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:19 smithi026 ceph-mon[86310]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:03:19.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:19 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:19.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:19 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:19.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:19 smithi026 ceph-mon[80969]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:03:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:19 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:19 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:19.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:19 smithi046 ceph-mon[65689]: pgmap v283: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:03:21.192 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:20 smithi046 ceph-mon[65689]: Upgrade: Updating rgw.foo.smithi046.wmzons (2/2) 2024-04-30T10:03:21.193 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:21.193 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T10:03:21.193 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:20 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:21.193 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:20 smithi046 ceph-mon[65689]: Deploying daemon rgw.foo.smithi046.wmzons on smithi046 2024-04-30T10:03:21.193 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:20 smithi046 ceph-mon[65689]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 85 B/s wr, 26 op/s 2024-04-30T10:03:21.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[86310]: Upgrade: Updating rgw.foo.smithi046.wmzons (2/2) 2024-04-30T10:03:21.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:21.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T10:03:21.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:21.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[86310]: Deploying daemon rgw.foo.smithi046.wmzons on smithi046 2024-04-30T10:03:21.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[86310]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 85 B/s wr, 26 op/s 2024-04-30T10:03:21.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[80969]: Upgrade: Updating rgw.foo.smithi046.wmzons (2/2) 2024-04-30T10:03:21.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:21.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.rgw.foo.smithi046.wmzons", "caps": ["mon", "allow *", "mgr", "allow rw", "osd", "allow rwx tag rgw *=*"]}]: dispatch 2024-04-30T10:03:21.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:21.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[80969]: Deploying daemon rgw.foo.smithi046.wmzons on smithi046 2024-04-30T10:03:21.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:20 smithi026 ceph-mon[80969]: pgmap v284: 161 pgs: 161 active+clean; 585 KiB data, 253 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 85 B/s wr, 26 op/s 2024-04-30T10:03:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:23 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:23.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:23 smithi046 ceph-mon[65689]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 85 B/s wr, 51 op/s 2024-04-30T10:03:23.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:23 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:23.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:23 smithi026 ceph-mon[86310]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 85 B/s wr, 51 op/s 2024-04-30T10:03:23.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:23 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:23.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:23 smithi026 ceph-mon[80969]: pgmap v285: 161 pgs: 161 active+clean; 585 KiB data, 257 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 85 B/s wr, 51 op/s 2024-04-30T10:03:24.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:24.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:24.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:24 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:24.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:24 smithi026 ceph-mon[86310]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 85 B/s wr, 87 op/s 2024-04-30T10:03:24.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:24.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:24.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:24 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:24.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:24 smithi026 ceph-mon[80969]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 85 B/s wr, 87 op/s 2024-04-30T10:03:24.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:24.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:24.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:24 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:24.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:24 smithi046 ceph-mon[65689]: pgmap v286: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 85 B/s wr, 87 op/s 2024-04-30T10:03:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:03:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:03:26] "GET /metrics HTTP/1.1" 200 38294 "" "Prometheus/2.43.0" 2024-04-30T10:03:27.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:27 smithi026 ceph-mon[86310]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 170 B/s wr, 100 op/s 2024-04-30T10:03:27.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:27 smithi026 ceph-mon[80969]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 170 B/s wr, 100 op/s 2024-04-30T10:03:27.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:27 smithi046 ceph-mon[65689]: pgmap v287: 161 pgs: 161 active+clean; 585 KiB data, 261 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 170 B/s wr, 100 op/s 2024-04-30T10:03:28.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:28 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:28 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:28 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:28 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:28 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:28 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:28 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:28 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:28 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:28 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:28 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:28.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:28 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:29 smithi046 ceph-mon[65689]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 170 B/s wr, 159 op/s 2024-04-30T10:03:29.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:29 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:29 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.611 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:29 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.612 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:29 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[86310]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 170 B/s wr, 159 op/s 2024-04-30T10:03:29.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[80969]: pgmap v288: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 170 B/s wr, 159 op/s 2024-04-30T10:03:29.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:29.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:29 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:31.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:31 smithi026 ceph-mon[86310]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-30T10:03:31.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:31.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:31 smithi026 ceph-mon[80969]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-30T10:03:31.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:31.839 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:31 smithi046 ceph-mon[65689]: pgmap v289: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 109 KiB/s rd, 170 B/s wr, 167 op/s 2024-04-30T10:03:31.839 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:33.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:33.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:33.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s rd, 86 B/s wr, 144 op/s 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 103 B/s wr, 143 op/s 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all rgw 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.igccwv"}]: dispatch 2024-04-30T10:03:33.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.igccwv"}]': finished 2024-04-30T10:03:33.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi046.wmzons"}]: dispatch 2024-04-30T10:03:33.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi046.wmzons"}]': finished 2024-04-30T10:03:33.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T10:03:33.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s rd, 86 B/s wr, 144 op/s 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 103 B/s wr, 143 op/s 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all rgw 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.igccwv"}]: dispatch 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.igccwv"}]': finished 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi046.wmzons"}]: dispatch 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi046.wmzons"}]': finished 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.452 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all nfs 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all nvmeof 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: Upgrade: Finalizing container_image settings 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-30T10:03:33.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all nfs 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all nvmeof 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: Upgrade: Finalizing container_image settings 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-30T10:03:33.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T10:03:33.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T10:03:33.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T10:03:33.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T10:03:33.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T10:03:33.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T10:03:33.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T10:03:33.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T10:03:33.457 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: Upgrade: Complete! 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:33.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 125 B/s wr, 119 op/s 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 0 B/s wr, 132 op/s 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: Checking dashboard <-> RGW credentials 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T10:03:33.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: Upgrade: Complete! 2024-04-30T10:03:33.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 125 B/s wr, 119 op/s 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 0 B/s wr, 132 op/s 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: Checking dashboard <-> RGW credentials 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:33.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:33 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: pgmap v290: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s rd, 86 B/s wr, 144 op/s 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: pgmap v291: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 93 KiB/s rd, 103 B/s wr, 143 op/s 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:33.485 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all rgw 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.igccwv"}]: dispatch 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi026.igccwv"}]': finished 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi046.wmzons"}]: dispatch 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw.foo.smithi046.wmzons"}]': finished 2024-04-30T10:03:33.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nfs 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all nvmeof 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.487 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: Upgrade: Finalizing container_image settings 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-04-30T10:03:33.488 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "osd"}]': finished 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mds"}]': finished 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-04-30T10:03:33.489 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: Upgrade: Complete! 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: pgmap v292: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 125 B/s wr, 119 op/s 2024-04-30T10:03:33.490 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: pgmap v293: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 0 B/s wr, 132 op/s 2024-04-30T10:03:33.491 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: Checking dashboard <-> RGW credentials 2024-04-30T10:03:33.491 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:33.491 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:03:33.491 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:03:33.491 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:03:33.491 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:33 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:34 smithi046 ceph-mon[65689]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 24 op/s 2024-04-30T10:03:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:34 smithi046 ceph-mon[65689]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-04-30T10:03:34.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:34 smithi026 ceph-mon[80969]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 24 op/s 2024-04-30T10:03:34.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:34 smithi026 ceph-mon[80969]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-04-30T10:03:34.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:34 smithi026 ceph-mon[86310]: pgmap v294: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 24 op/s 2024-04-30T10:03:34.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:34 smithi026 ceph-mon[86310]: pgmap v295: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 2.4 KiB/s rd, 2 op/s 2024-04-30T10:03:36.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:36 smithi046 ceph-mon[65689]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 5 op/s 2024-04-30T10:03:36.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:36 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:36.698 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:03:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:03:36] "GET /metrics HTTP/1.1" 200 38329 "" "Prometheus/2.43.0" 2024-04-30T10:03:36.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:36 smithi026 ceph-mon[80969]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 5 op/s 2024-04-30T10:03:36.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:36 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:36.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:36 smithi026 ceph-mon[86310]: pgmap v296: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 5 op/s 2024-04-30T10:03:36.699 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:36 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:03:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:38 smithi046 ceph-mon[65689]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-04-30T10:03:38.498 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:38 smithi026 ceph-mon[80969]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-04-30T10:03:38.499 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:38 smithi026 ceph-mon[86310]: pgmap v297: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 4 op/s 2024-04-30T10:03:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:40 smithi046 ceph-mon[65689]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 4 op/s 2024-04-30T10:03:40.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:40 smithi026 ceph-mon[80969]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 4 op/s 2024-04-30T10:03:40.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:40 smithi026 ceph-mon[86310]: pgmap v298: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 4 op/s 2024-04-30T10:03:42.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:42 smithi026 ceph-mon[80969]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-04-30T10:03:42.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:42 smithi026 ceph-mon[86310]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-04-30T10:03:42.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:42 smithi046 ceph-mon[65689]: pgmap v299: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 3 op/s 2024-04-30T10:03:43.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:43 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:43.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:43 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:43.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:43 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:44.208 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T10:03:44.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:44 smithi026 ceph-mon[80969]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-30T10:03:44.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:44 smithi026 ceph-mon[80969]: from='client.54661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:44 smithi026 ceph-mon[86310]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-30T10:03:44.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:44 smithi026 ceph-mon[86310]: from='client.54661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:44 smithi046 ceph-mon[65689]: pgmap v300: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 3 op/s 2024-04-30T10:03:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:44 smithi046 ceph-mon[65689]: from='client.54661 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:46.362 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:03:46.362 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (9m) 18s ago 32m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:03:46.362 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (9m) 18s ago 32m 82.5M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:03:46.362 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 18s ago 31m 64.5M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (11m) 18s ago 36m 456M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (21m) 18s ago 38m 512M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (7m) 18s ago 38m 62.0M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (8m) 18s ago 37m 48.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (6m) 18s ago 37m 51.3M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (10m) 18s ago 32m 16.3M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (10m) 18s ago 32m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (5m) 18s ago 36m 55.6M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (4m) 18s ago 36m 54.2M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (3m) 18s ago 35m 48.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (2m) 18s ago 35m 62.2M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 8d72616b503d 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (2m) 18s ago 34m 57.2M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a 501e0c080dad 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (2m) 18s ago 34m 47.8M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b070c14531b0 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (98s) 18s ago 34m 47.4M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a ce35c1eb8525 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (78s) 18s ago 33m 53.7M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b081d959354e 2024-04-30T10:03:46.363 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (10m) 18s ago 33m 80.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:03:46.364 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (28s) 18s ago 31m 77.2M - 19.0.0-3357-g3a873b0c c50cf5fa633a cc20e54b564a 2024-04-30T10:03:46.364 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (23s) 18s ago 31m 73.9M - 19.0.0-3357-g3a873b0c c50cf5fa633a a868ebccfdb5 2024-04-30T10:03:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:46 smithi046 ceph-mon[65689]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2 op/s 2024-04-30T10:03:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:46.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:03:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:03:46] "GET /metrics HTTP/1.1" 200 38327 "" "Prometheus/2.43.0" 2024-04-30T10:03:46.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:46 smithi026 ceph-mon[80969]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2 op/s 2024-04-30T10:03:46.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:46.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:46 smithi026 ceph-mon[86310]: pgmap v301: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 2 op/s 2024-04-30T10:03:46.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:03:47.035 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.rgw | length == 1'"'"'' 2024-04-30T10:03:47.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:47 smithi026 ceph-mon[86310]: from='client.44736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:47.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:47 smithi026 ceph-mon[80969]: from='client.44736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:47.486 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:47 smithi046 ceph-mon[65689]: from='client.44736 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:48.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:48 smithi026 ceph-mon[86310]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-04-30T10:03:48.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:48 smithi026 ceph-mon[80969]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-04-30T10:03:48.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:48 smithi046 ceph-mon[65689]: pgmap v302: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 2 op/s 2024-04-30T10:03:49.242 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:03:49.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:49 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/840587559' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:49.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:49 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/840587559' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:49.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:49 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/840587559' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:49.983 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.rgw | keys'"'"' | grep $sha1' 2024-04-30T10:03:50.502 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:50 smithi026 ceph-mon[80969]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-30T10:03:50.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:50 smithi026 ceph-mon[86310]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-30T10:03:50.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:50 smithi046 ceph-mon[65689]: pgmap v303: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s rd, 1 op/s 2024-04-30T10:03:51.968 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)" 2024-04-30T10:03:52.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:52 smithi026 ceph-mon[86310]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:52.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:52 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/795816899' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:52.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:52 smithi026 ceph-mon[80969]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:52.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:52 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/795816899' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:52.681 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade status' 2024-04-30T10:03:52.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:52 smithi046 ceph-mon[65689]: pgmap v304: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:52.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:52 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/795816899' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:03:53.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:53 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:53.629 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:53 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:53.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:53 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:03:54.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:54 smithi026 ceph-mon[80969]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:54.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:54 smithi026 ceph-mon[86310]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:54.618 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:03:54.618 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": null, 2024-04-30T10:03:54.618 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": false, 2024-04-30T10:03:54.618 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "", 2024-04-30T10:03:54.618 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [], 2024-04-30T10:03:54.618 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": null, 2024-04-30T10:03:54.618 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "", 2024-04-30T10:03:54.618 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T10:03:54.618 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:03:54.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:54 smithi046 ceph-mon[65689]: pgmap v305: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:55.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T10:03:56.537 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:56 smithi026 ceph-mon[86310]: from='client.54685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:56.537 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:56 smithi026 ceph-mon[86310]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:56.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:56 smithi026 ceph-mon[80969]: from='client.54685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:56.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:56 smithi026 ceph-mon[80969]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:56.538 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:03:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:03:56] "GET /metrics HTTP/1.1" 200 38327 "" "Prometheus/2.43.0" 2024-04-30T10:03:56.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:56 smithi046 ceph-mon[65689]: from='client.54685 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:03:56.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:56 smithi046 ceph-mon[65689]: pgmap v306: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:03:57.161 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:03:57.161 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:03:57.161 INFO:teuthology.orchestra.run.smithi026.stdout: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T10:03:57.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:57 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/3195500465' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:03:57.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:57 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/3195500465' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:03:57.561 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:57 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/3195500465' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:03:57.938 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-30T10:03:58.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:03:58 smithi026 ceph-mon[86310]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:58.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:03:58 smithi026 ceph-mon[80969]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:58.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:03:58 smithi046 ceph-mon[65689]: pgmap v307: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:03:59.814 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:04:00.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[86310]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:00.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:00.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:04:00.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:04:00.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:04:00.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:00.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[80969]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:00.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:00.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:04:00.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:04:00.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:04:00.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:00 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:00.454 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-30T10:04:00.464 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2024-04-30T10:04:00.464 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done' 2024-04-30T10:04:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:00 smithi046 ceph-mon[65689]: pgmap v308: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:04:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:04:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:04:00.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:00 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:01.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[86310]: from='client.54697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[86310]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[86310]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[86310]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[86310]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[80969]: from='client.54697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[80969]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[80969]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[80969]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:01.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[80969]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:04:01.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:01 smithi046 ceph-mon[65689]: from='client.54697 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:01 smithi046 ceph-mon[65689]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:04:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:01 smithi046 ceph-mon[65689]: pgmap v309: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 933 B/s rd, 0 op/s 2024-04-30T10:04:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:01 smithi046 ceph-mon[65689]: pgmap v310: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:01 smithi046 ceph-mon[65689]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:3a873b0ce83192ee05b56734dad1076a7a94ecc7 2024-04-30T10:04:01.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:03.020 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:04:03.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:03 smithi026 ceph-mon[86310]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:03.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:03 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:03.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:03 smithi026 ceph-mon[80969]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:03.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:03 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:03.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:03 smithi046 ceph-mon[65689]: pgmap v311: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:03.735 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:03 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:03.735 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:04:03.735 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (10m) 36s ago 32m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:04:03.735 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (9m) 36s ago 32m 82.5M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:04:03.735 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 36s ago 31m 64.5M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:04:03.735 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (11m) 36s ago 37m 456M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:04:03.735 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (21m) 36s ago 38m 512M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (7m) 36s ago 39m 62.0M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (8m) 36s ago 37m 48.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (7m) 36s ago 37m 51.3M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (11m) 36s ago 33m 16.3M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (10m) 36s ago 33m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (5m) 36s ago 36m 55.6M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (5m) 36s ago 36m 54.2M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (4m) 36s ago 36m 48.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (3m) 36s ago 35m 62.2M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 8d72616b503d 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (2m) 36s ago 35m 57.2M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a 501e0c080dad 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (2m) 36s ago 34m 47.8M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b070c14531b0 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (115s) 36s ago 34m 47.4M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a ce35c1eb8525 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (95s) 36s ago 34m 53.7M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b081d959354e 2024-04-30T10:04:03.736 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (10m) 36s ago 33m 80.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:04:03.737 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (45s) 36s ago 31m 77.2M - 19.0.0-3357-g3a873b0c c50cf5fa633a cc20e54b564a 2024-04-30T10:04:03.737 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (40s) 36s ago 31m 73.9M - 19.0.0-3357-g3a873b0c c50cf5fa633a a868ebccfdb5 2024-04-30T10:04:04.213 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:04:04.213 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T10:04:04.213 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T10:04:04.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:04:04.213 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T10:04:04.213 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:04:04.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:04:04.213 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T10:04:04.213 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 8 2024-04-30T10:04:04.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:04:04.214 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T10:04:04.214 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:04:04.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:04:04.214 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T10:04:04.214 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 15 2024-04-30T10:04:04.214 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T10:04:04.214 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:04:04.590 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:04:04.590 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T10:04:04.590 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T10:04:04.590 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr", 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw", 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout: "osd" 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "15/21 daemons upgraded", 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Currently upgrading iscsi daemons", 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": false 2024-04-30T10:04:04.591 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='client.54703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='client.44778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mgr 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.763 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mon 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all crash 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all osd 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all mds 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all rgw 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/298199981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='client.54703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='client.44778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mgr 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mon 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all crash 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.766 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all osd 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all mds 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all rgw 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T10:04:04.768 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.768 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:04 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/298199981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='client.54703 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='client.44778 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Target is version 19.0.0-3357-g3a873b0c (squid) 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d'] 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mgr 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mon 2024-04-30T10:04:04.985 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all crash 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all osd 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all mds 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all rgw 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.986 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all rbd-mirror 2024-04-30T10:04:04.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all cephfs-mirror 2024-04-30T10:04:04.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:04.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: Upgrade: Setting container_image for all ceph-exporter 2024-04-30T10:04:04.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:04.987 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:04 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/298199981' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:05.008 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:04:05.008 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:04:05.008 INFO:teuthology.orchestra.run.smithi026.stdout: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T10:04:05.830 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[86310]: from='client.54715 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[86310]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[86310]: from='client.54727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/1626489857' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[80969]: from='client.54715 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[80969]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[80969]: from='client.54727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/1626489857' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:05.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T10:04:05.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:05 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:04:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:05 smithi046 ceph-mon[65689]: from='client.54715 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:05 smithi046 ceph-mon[65689]: pgmap v312: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:05 smithi046 ceph-mon[65689]: from='client.54727 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:05 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/1626489857' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:04:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "client.iscsi.foo.smithi026.romfrr", "caps": ["mon", "profile rbd, allow command \"osd blocklist\", allow command \"config-key get\" with \"key\" prefix \"iscsi/\"", "mgr", "allow command \"service status\"", "osd", "allow rwx"]}]: dispatch 2024-04-30T10:04:05.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:05 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:04:06.571 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:04:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:04:06] "GET /metrics HTTP/1.1" 200 38327 "" "Prometheus/2.43.0" 2024-04-30T10:04:06.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:06 smithi026 ceph-mon[86310]: Upgrade: Updating iscsi.foo.smithi026.romfrr 2024-04-30T10:04:06.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:06 smithi026 ceph-mon[86310]: Deploying daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T10:04:06.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:06 smithi026 ceph-mon[80969]: Upgrade: Updating iscsi.foo.smithi026.romfrr 2024-04-30T10:04:06.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:06 smithi026 ceph-mon[80969]: Deploying daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T10:04:06.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:06 smithi046 ceph-mon[65689]: Upgrade: Updating iscsi.foo.smithi026.romfrr 2024-04-30T10:04:06.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:06 smithi046 ceph-mon[65689]: Deploying daemon iscsi.foo.smithi026.romfrr on smithi026 2024-04-30T10:04:07.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:07 smithi026 ceph-mon[86310]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 571 B/s rd, 0 op/s 2024-04-30T10:04:07.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:07 smithi026 ceph-mon[80969]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 571 B/s rd, 0 op/s 2024-04-30T10:04:07.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:07 smithi046 ceph-mon[65689]: pgmap v313: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 571 B/s rd, 0 op/s 2024-04-30T10:04:09.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:09 smithi026 ceph-mon[86310]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:09.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:09 smithi026 ceph-mon[80969]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:09.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:09 smithi046 ceph-mon[65689]: pgmap v314: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:04:11.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:11 smithi026 ceph-mon[80969]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:04:11.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:11 smithi026 ceph-mon[86310]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:04:11.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:11 smithi046 ceph-mon[65689]: pgmap v315: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:04:13.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:13 smithi026 ceph-mon[86310]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:13.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:13 smithi026 ceph-mon[86310]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:13 smithi026 ceph-mon[80969]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:13.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:13 smithi026 ceph-mon[80969]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:13.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:13 smithi046 ceph-mon[65689]: pgmap v316: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:13.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:13 smithi046 ceph-mon[65689]: from='client.15378 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:15.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:15 smithi026 ceph-mon[86310]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:15.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:15 smithi026 ceph-mon[80969]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:15.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:15 smithi046 ceph-mon[65689]: pgmap v317: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:16.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:16.697 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:04:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:04:16] "GET /metrics HTTP/1.1" 200 38328 "" "Prometheus/2.43.0" 2024-04-30T10:04:16.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:16.885 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:17.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:17 smithi026 ceph-mon[86310]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:17.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:17 smithi026 ceph-mon[80969]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:17.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:17 smithi046 ceph-mon[65689]: pgmap v318: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:18.002 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:04:17 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T10:04:17.840+0000 7f3e38788700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed. 2024-04-30T10:04:18.734 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=cleanup t=2024-04-30T10:04:18.419905863Z level=info msg="Completed cleanup jobs" duration=2.341401ms 2024-04-30T10:04:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed. 2024-04-30T10:04:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:04:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:04:19.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[80969]: Health check failed: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed. 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:04:19.198 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:04:19.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:19.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-04-30T10:04:19.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:19.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:18 smithi026 ceph-mon[86310]: Health check failed: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-30T10:04:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: Upgrade: Paused due to UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed. 2024-04-30T10:04:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: pgmap v319: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: pgmap v320: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:04:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:04:19.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:04:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:04:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: pgmap v321: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: pgmap v322: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-04-30T10:04:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:19.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:18 smithi046 ceph-mon[65689]: Health check failed: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed. (UPGRADE_REDEPLOY_DAEMON) 2024-04-30T10:04:21.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:20 smithi026 ceph-mon[86310]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-04-30T10:04:21.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:20 smithi026 ceph-mon[80969]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-04-30T10:04:21.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:20 smithi046 ceph-mon[65689]: pgmap v323: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 851 B/s rd, 0 op/s 2024-04-30T10:04:23.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:22 smithi026 ceph-mon[86310]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-04-30T10:04:23.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:22 smithi026 ceph-mon[80969]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-04-30T10:04:23.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:22 smithi046 ceph-mon[65689]: pgmap v324: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1022 B/s rd, 0 op/s 2024-04-30T10:04:25.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:24 smithi026 ceph-mon[86310]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-30T10:04:25.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:24 smithi026 ceph-mon[80969]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-30T10:04:25.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:24 smithi046 ceph-mon[65689]: pgmap v325: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 170 B/s rd, 0 op/s 2024-04-30T10:04:26.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:04:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:04:26] "GET /metrics HTTP/1.1" 200 38328 "" "Prometheus/2.43.0" 2024-04-30T10:04:27.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:26 smithi026 ceph-mon[86310]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-30T10:04:27.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:26 smithi026 ceph-mon[80969]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-30T10:04:27.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:26 smithi046 ceph-mon[65689]: pgmap v326: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 127 B/s rd, 0 op/s 2024-04-30T10:04:29.060 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:28 smithi026 ceph-mon[80969]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-30T10:04:29.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:28 smithi026 ceph-mon[86310]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-30T10:04:29.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:28 smithi046 ceph-mon[65689]: pgmap v327: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 102 B/s rd, 0 op/s 2024-04-30T10:04:30.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:29 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/1380718388' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T10:04:30.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:29 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2239838970' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/869253470"}]: dispatch 2024-04-30T10:04:30.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:29 smithi026 ceph-mon[86310]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/869253470"}]: dispatch 2024-04-30T10:04:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:29 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/1380718388' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T10:04:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:29 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2239838970' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/869253470"}]: dispatch 2024-04-30T10:04:30.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:29 smithi026 ceph-mon[80969]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/869253470"}]: dispatch 2024-04-30T10:04:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:29 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/1380718388' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist ls"}]: dispatch 2024-04-30T10:04:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:29 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2239838970' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/869253470"}]: dispatch 2024-04-30T10:04:30.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:29 smithi046 ceph-mon[65689]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/869253470"}]: dispatch 2024-04-30T10:04:31.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[80969]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-30T10:04:31.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[80969]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/869253470"}]': finished 2024-04-30T10:04:31.196 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[80969]: osdmap e155: 8 total, 8 up, 8 in 2024-04-30T10:04:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/3903642530' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/983594036"}]: dispatch 2024-04-30T10:04:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:31.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[86310]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-30T10:04:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[86310]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/869253470"}]': finished 2024-04-30T10:04:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[86310]: osdmap e155: 8 total, 8 up, 8 in 2024-04-30T10:04:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/3903642530' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/983594036"}]: dispatch 2024-04-30T10:04:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:31.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:30 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:30 smithi046 ceph-mon[65689]: pgmap v328: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 85 B/s rd, 0 op/s 2024-04-30T10:04:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:30 smithi046 ceph-mon[65689]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/869253470"}]': finished 2024-04-30T10:04:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:30 smithi046 ceph-mon[65689]: osdmap e155: 8 total, 8 up, 8 in 2024-04-30T10:04:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:30 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/3903642530' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/983594036"}]: dispatch 2024-04-30T10:04:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:04:31.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:30 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:31 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/3903642530' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/983594036"}]': finished 2024-04-30T10:04:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:31 smithi026 ceph-mon[86310]: osdmap e156: 8 total, 8 up, 8 in 2024-04-30T10:04:32.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:31 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/946513889' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4056779445"}]: dispatch 2024-04-30T10:04:32.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:31 smithi026 ceph-mon[86310]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4056779445"}]: dispatch 2024-04-30T10:04:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:31 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/3903642530' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/983594036"}]': finished 2024-04-30T10:04:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:31 smithi026 ceph-mon[80969]: osdmap e156: 8 total, 8 up, 8 in 2024-04-30T10:04:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:31 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/946513889' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4056779445"}]: dispatch 2024-04-30T10:04:32.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:31 smithi026 ceph-mon[80969]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4056779445"}]: dispatch 2024-04-30T10:04:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:31 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/3903642530' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/983594036"}]': finished 2024-04-30T10:04:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:31 smithi046 ceph-mon[65689]: osdmap e156: 8 total, 8 up, 8 in 2024-04-30T10:04:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:31 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/946513889' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4056779445"}]: dispatch 2024-04-30T10:04:32.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:31 smithi046 ceph-mon[65689]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4056779445"}]: dispatch 2024-04-30T10:04:33.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[86310]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-30T10:04:33.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[86310]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4056779445"}]': finished 2024-04-30T10:04:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[86310]: osdmap e157: 8 total, 8 up, 8 in 2024-04-30T10:04:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/904014495' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1994817051"}]: dispatch 2024-04-30T10:04:33.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[86310]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1994817051"}]: dispatch 2024-04-30T10:04:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[80969]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-30T10:04:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[80969]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4056779445"}]': finished 2024-04-30T10:04:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[80969]: osdmap e157: 8 total, 8 up, 8 in 2024-04-30T10:04:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/904014495' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1994817051"}]: dispatch 2024-04-30T10:04:33.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:32 smithi026 ceph-mon[80969]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1994817051"}]: dispatch 2024-04-30T10:04:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:32 smithi046 ceph-mon[65689]: pgmap v331: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 383 B/s rd, 0 op/s 2024-04-30T10:04:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:32 smithi046 ceph-mon[65689]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/4056779445"}]': finished 2024-04-30T10:04:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:32 smithi046 ceph-mon[65689]: osdmap e157: 8 total, 8 up, 8 in 2024-04-30T10:04:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:32 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/904014495' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1994817051"}]: dispatch 2024-04-30T10:04:33.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:32 smithi046 ceph-mon[65689]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1994817051"}]: dispatch 2024-04-30T10:04:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:33 smithi046 ceph-mon[65689]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1994817051"}]': finished 2024-04-30T10:04:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:33 smithi046 ceph-mon[65689]: osdmap e158: 8 total, 8 up, 8 in 2024-04-30T10:04:34.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:33 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/1322823193' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/3239738018"}]: dispatch 2024-04-30T10:04:34.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:33 smithi026 ceph-mon[86310]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1994817051"}]': finished 2024-04-30T10:04:34.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:33 smithi026 ceph-mon[86310]: osdmap e158: 8 total, 8 up, 8 in 2024-04-30T10:04:34.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:33 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/1322823193' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/3239738018"}]: dispatch 2024-04-30T10:04:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:33 smithi026 ceph-mon[80969]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:0/1994817051"}]': finished 2024-04-30T10:04:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:33 smithi026 ceph-mon[80969]: osdmap e158: 8 total, 8 up, 8 in 2024-04-30T10:04:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:33 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/1322823193' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/3239738018"}]: dispatch 2024-04-30T10:04:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:34 smithi046 ceph-mon[65689]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T10:04:35.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:34 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/1322823193' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/3239738018"}]': finished 2024-04-30T10:04:35.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:34 smithi046 ceph-mon[65689]: osdmap e159: 8 total, 8 up, 8 in 2024-04-30T10:04:35.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:34 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2931796930' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/3239738018"}]: dispatch 2024-04-30T10:04:35.235 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:34 smithi046 ceph-mon[65689]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/3239738018"}]: dispatch 2024-04-30T10:04:35.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[86310]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T10:04:35.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/1322823193' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/3239738018"}]': finished 2024-04-30T10:04:35.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[86310]: osdmap e159: 8 total, 8 up, 8 in 2024-04-30T10:04:35.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2931796930' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/3239738018"}]: dispatch 2024-04-30T10:04:35.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[86310]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/3239738018"}]: dispatch 2024-04-30T10:04:35.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[80969]: pgmap v334: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 0 op/s 2024-04-30T10:04:35.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/1322823193' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6801/3239738018"}]': finished 2024-04-30T10:04:35.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[80969]: osdmap e159: 8 total, 8 up, 8 in 2024-04-30T10:04:35.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2931796930' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/3239738018"}]: dispatch 2024-04-30T10:04:35.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:34 smithi026 ceph-mon[80969]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/3239738018"}]: dispatch 2024-04-30T10:04:35.380 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:04:35.730 INFO:teuthology.orchestra.run.smithi026.stdout:"Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed." 2024-04-30T10:04:36.030 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:35 smithi026 ceph-mon[86310]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/3239738018"}]': finished 2024-04-30T10:04:36.030 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:35 smithi026 ceph-mon[86310]: osdmap e160: 8 total, 8 up, 8 in 2024-04-30T10:04:36.031 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:35 smithi026 ceph-mon[86310]: from='client.54799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:36.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:35 smithi026 ceph-mon[80969]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/3239738018"}]': finished 2024-04-30T10:04:36.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:35 smithi026 ceph-mon[80969]: osdmap e160: 8 total, 8 up, 8 in 2024-04-30T10:04:36.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:35 smithi026 ceph-mon[80969]: from='client.54799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:35 smithi046 ceph-mon[65689]: from='client.? ' entity='client.iscsi.foo.smithi026.romfrr' cmd='[{"prefix": "osd blocklist", "blocklistop": "rm", "addr": "172.21.15.26:6800/3239738018"}]': finished 2024-04-30T10:04:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:35 smithi046 ceph-mon[65689]: osdmap e160: 8 total, 8 up, 8 in 2024-04-30T10:04:36.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:35 smithi046 ceph-mon[65689]: from='client.54799 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:36.388 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T10:04:36.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:04:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:04:36] "GET /metrics HTTP/1.1" 200 38408 "" "Prometheus/2.43.0" 2024-04-30T10:04:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:36 smithi046 ceph-mon[65689]: from='client.54805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:37.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:36 smithi046 ceph-mon[65689]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:04:37.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:36 smithi026 ceph-mon[86310]: from='client.54805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:37.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:36 smithi026 ceph-mon[86310]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:04:37.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:36 smithi026 ceph-mon[80969]: from='client.54805 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:37.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:36 smithi026 ceph-mon[80969]: pgmap v337: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail 2024-04-30T10:04:38.500 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:04:38.500 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (10m) 70s ago 33m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (10m) 70s ago 32m 82.5M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 70s ago 32m 64.5M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (12m) 70s ago 37m 456M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (22m) 70s ago 39m 512M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (8m) 70s ago 39m 62.0M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (9m) 70s ago 38m 48.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (7m) 70s ago 38m 51.3M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (11m) 70s ago 33m 16.3M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (11m) 70s ago 33m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (6m) 70s ago 37m 55.6M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (5m) 70s ago 37m 54.2M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (4m) 70s ago 36m 48.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (3m) 70s ago 36m 62.2M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 8d72616b503d 2024-04-30T10:04:38.501 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (3m) 70s ago 35m 57.2M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a 501e0c080dad 2024-04-30T10:04:38.502 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (3m) 70s ago 35m 47.8M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b070c14531b0 2024-04-30T10:04:38.502 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (2m) 70s ago 35m 47.4M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a ce35c1eb8525 2024-04-30T10:04:38.502 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (2m) 70s ago 34m 53.7M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b081d959354e 2024-04-30T10:04:38.502 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (11m) 70s ago 33m 80.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:04:38.502 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (80s) 70s ago 32m 77.2M - 19.0.0-3357-g3a873b0c c50cf5fa633a cc20e54b564a 2024-04-30T10:04:38.502 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (75s) 70s ago 32m 73.9M - 19.0.0-3357-g3a873b0c c50cf5fa633a a868ebccfdb5 2024-04-30T10:04:39.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:38 smithi026 ceph-mon[86310]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-04-30T10:04:39.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:38 smithi026 ceph-mon[86310]: from='client.44880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:39.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:38 smithi026 ceph-mon[80969]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-04-30T10:04:39.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:38 smithi026 ceph-mon[80969]: from='client.44880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:39.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:38 smithi046 ceph-mon[65689]: pgmap v338: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 864 B/s rd, 0 op/s 2024-04-30T10:04:39.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:38 smithi046 ceph-mon[65689]: from='client.44880 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:04:39.241 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions' 2024-04-30T10:04:40.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:39 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:40.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:39 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:40.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:39 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:41.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:40 smithi046 ceph-mon[65689]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 739 B/s rd, 0 op/s 2024-04-30T10:04:41.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:40 smithi026 ceph-mon[86310]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 739 B/s rd, 0 op/s 2024-04-30T10:04:41.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:40 smithi026 ceph-mon[80969]: pgmap v339: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 739 B/s rd, 0 op/s 2024-04-30T10:04:41.503 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:04:41.503 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T10:04:41.503 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T10:04:41.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:04:41.503 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 8 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 15 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T10:04:41.504 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:04:42.196 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:41 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2605530720' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:42.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:41 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2605530720' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:42.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'echo "wait for servicemap items w/ changing names to refresh"' 2024-04-30T10:04:42.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:41 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2605530720' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:04:43.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:42 smithi026 ceph-mon[86310]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:43.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:42 smithi026 ceph-mon[80969]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:43.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:42 smithi046 ceph-mon[65689]: pgmap v340: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:43.964 INFO:teuthology.orchestra.run.smithi026.stdout:wait for servicemap items w/ changing names to refresh 2024-04-30T10:04:44.665 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'sleep 60' 2024-04-30T10:04:45.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:44 smithi046 ceph-mon[65689]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T10:04:45.305 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:44 smithi026 ceph-mon[86310]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T10:04:45.305 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:44 smithi026 ceph-mon[80969]: pgmap v341: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1 op/s 2024-04-30T10:04:46.223 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:45 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:46.223 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:45 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:46.234 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:45 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:04:46.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:04:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:04:46] "GET /metrics HTTP/1.1" 200 38407 "" "Prometheus/2.43.0" 2024-04-30T10:04:47.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:46 smithi026 ceph-mon[80969]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-30T10:04:47.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:46 smithi026 ceph-mon[86310]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-30T10:04:47.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:46 smithi046 ceph-mon[65689]: pgmap v342: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 938 B/s rd, 0 op/s 2024-04-30T10:04:49.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:48 smithi026 ceph-mon[86310]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:49.127 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:48 smithi026 ceph-mon[80969]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:49.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:48 smithi046 ceph-mon[65689]: pgmap v343: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:50.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:49 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:50.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:49 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:49 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:04:51.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:50 smithi026 ceph-mon[86310]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:04:51.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:50 smithi026 ceph-mon[80969]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:04:51.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:50 smithi046 ceph-mon[65689]: pgmap v344: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:04:53.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:52 smithi026 ceph-mon[86310]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:53.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:52 smithi026 ceph-mon[80969]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:53.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:52 smithi046 ceph-mon[65689]: pgmap v345: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:55.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:55 smithi026 ceph-mon[86310]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:55.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:55 smithi026 ceph-mon[80969]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:55.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:55 smithi046 ceph-mon[65689]: pgmap v346: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:04:56.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:04:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:04:56] "GET /metrics HTTP/1.1" 200 38407 "" "Prometheus/2.43.0" 2024-04-30T10:04:57.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:57 smithi026 ceph-mon[80969]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:04:57.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:57 smithi026 ceph-mon[86310]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:04:57.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:57 smithi046 ceph-mon[65689]: pgmap v347: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:04:59.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:04:59 smithi026 ceph-mon[80969]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:59.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:04:59 smithi026 ceph-mon[86310]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:04:59.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:04:59 smithi046 ceph-mon[65689]: pgmap v348: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:00.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:00 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:00.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:00 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:00 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:01.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:01 smithi026 ceph-mon[86310]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:01.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:01.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:01 smithi026 ceph-mon[80969]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:01.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:01 smithi046 ceph-mon[65689]: pgmap v349: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:03.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:03 smithi026 ceph-mon[80969]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:03.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:03 smithi026 ceph-mon[86310]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:03 smithi046 ceph-mon[65689]: pgmap v350: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:05.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:05 smithi026 ceph-mon[80969]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:05.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:05 smithi026 ceph-mon[86310]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:05.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:05 smithi046 ceph-mon[65689]: pgmap v351: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:06.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:05:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:05:06] "GET /metrics HTTP/1.1" 200 38408 "" "Prometheus/2.43.0" 2024-04-30T10:05:07.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:07 smithi026 ceph-mon[80969]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:07 smithi026 ceph-mon[86310]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:07 smithi046 ceph-mon[65689]: pgmap v352: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:09.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:09 smithi026 ceph-mon[86310]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:09.386 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:09 smithi026 ceph-mon[80969]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:09 smithi046 ceph-mon[65689]: pgmap v353: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:10.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:10 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:10.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:10 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:10.483 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:10 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:11.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:11 smithi026 ceph-mon[80969]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:11.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:11 smithi026 ceph-mon[86310]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:11.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:11 smithi046 ceph-mon[65689]: pgmap v354: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:13.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:13 smithi026 ceph-mon[80969]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:13.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:13 smithi026 ceph-mon[86310]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:13.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:13 smithi046 ceph-mon[65689]: pgmap v355: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:15.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:15 smithi026 ceph-mon[80969]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:15.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:15 smithi026 ceph-mon[86310]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:15.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:15 smithi046 ceph-mon[65689]: pgmap v356: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:16.395 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:16.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:16.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:05:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:05:16] "GET /metrics HTTP/1.1" 200 38408 "" "Prometheus/2.43.0" 2024-04-30T10:05:17.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:17 smithi026 ceph-mon[86310]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:17.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:17 smithi026 ceph-mon[80969]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:17.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:17 smithi046 ceph-mon[65689]: pgmap v357: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:18 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:05:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:18 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:05:18.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:18 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-30T10:05:19.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:19 smithi026 ceph-mon[86310]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:19 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:05:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:19 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:05:19.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:19 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:05:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:19 smithi026 ceph-mon[80969]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:19 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:05:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:19 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:05:19.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:19 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:05:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:19 smithi046 ceph-mon[65689]: pgmap v358: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:19 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-30T10:05:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:19 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-30T10:05:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:19 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' 2024-04-30T10:05:20.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:20 smithi026 ceph-mon[80969]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-04-30T10:05:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:20 smithi026 ceph-mon[80969]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:20.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:20 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:20 smithi026 ceph-mon[86310]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-04-30T10:05:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:20 smithi026 ceph-mon[86310]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:20.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:20 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:20 smithi046 ceph-mon[65689]: pgmap v359: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 948 B/s rd, 0 op/s 2024-04-30T10:05:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:20 smithi046 ceph-mon[65689]: pgmap v360: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:20.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:20 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:22.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:22 smithi026 ceph-mon[86310]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-30T10:05:22.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:22 smithi026 ceph-mon[80969]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-30T10:05:22.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:22 smithi046 ceph-mon[65689]: pgmap v361: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 582 B/s rd, 0 op/s 2024-04-30T10:05:24.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:24 smithi026 ceph-mon[86310]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:24.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:24 smithi026 ceph-mon[80969]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:24.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:24 smithi046 ceph-mon[65689]: pgmap v362: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:26.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:26 smithi026 ceph-mon[86310]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:26.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:26 smithi026 ceph-mon[80969]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:26.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:05:26 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:05:26] "GET /metrics HTTP/1.1" 200 38408 "" "Prometheus/2.43.0" 2024-04-30T10:05:26.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:26 smithi046 ceph-mon[65689]: pgmap v363: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:28.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:28 smithi026 ceph-mon[86310]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:28.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:28 smithi026 ceph-mon[80969]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:28.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:28 smithi046 ceph-mon[65689]: pgmap v364: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s rd, 1 op/s 2024-04-30T10:05:30.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:30 smithi026 ceph-mon[80969]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:05:30.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:30 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:30.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:30 smithi026 ceph-mon[86310]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:05:30.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:30 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:30.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:30 smithi046 ceph-mon[65689]: pgmap v365: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 op/s 2024-04-30T10:05:30.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:30 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:31.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:31 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:31.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:31 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:31.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:31 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:32.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:32 smithi026 ceph-mon[80969]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:32.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:32 smithi026 ceph-mon[86310]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:32.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:32 smithi046 ceph-mon[65689]: pgmap v366: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:34.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:34 smithi026 ceph-mon[86310]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:34.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:34 smithi026 ceph-mon[80969]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:34.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:34 smithi046 ceph-mon[65689]: pgmap v367: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:36.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:36 smithi026 ceph-mon[86310]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:36.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:36 smithi026 ceph-mon[80969]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:36.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:05:36 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:05:36] "GET /metrics HTTP/1.1" 200 38404 "" "Prometheus/2.43.0" 2024-04-30T10:05:36.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:36 smithi046 ceph-mon[65689]: pgmap v368: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:38.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:38 smithi026 ceph-mon[86310]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:38.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:38 smithi026 ceph-mon[80969]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:38.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:38 smithi046 ceph-mon[65689]: pgmap v369: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:40.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:40 smithi026 ceph-mon[86310]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:40.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:40 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:40.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:40 smithi026 ceph-mon[80969]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:40.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:40 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:40 smithi046 ceph-mon[65689]: pgmap v370: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:05:40.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:40 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:42.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:42 smithi026 ceph-mon[86310]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:42.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:42 smithi026 ceph-mon[80969]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:42.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:42 smithi046 ceph-mon[65689]: pgmap v371: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:44.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:44 smithi026 ceph-mon[86310]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:44.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:44 smithi026 ceph-mon[80969]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:44.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:44 smithi046 ceph-mon[65689]: pgmap v372: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:46.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:46 smithi026 ceph-mon[86310]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:46.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:46 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:46.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:05:46 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:05:46] "GET /metrics HTTP/1.1" 200 38404 "" "Prometheus/2.43.0" 2024-04-30T10:05:46.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:46 smithi026 ceph-mon[80969]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:46.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:46 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:46 smithi046 ceph-mon[65689]: pgmap v373: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:46.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:46 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:05:47.111 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ps' 2024-04-30T10:05:48.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:48 smithi026 ceph-mon[80969]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:48.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:48 smithi026 ceph-mon[86310]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:48.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:48 smithi046 ceph-mon[65689]: pgmap v374: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:49.184 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-30T10:05:49.184 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (11m) 2m ago 34m 25.3M - 0.25.0 c8568f914cd2 1485dc9c824f 2024-04-30T10:05:49.184 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi046 *:3000 running (11m) 2m ago 34m 82.5M - 9.4.12 2bacad6d85d8 a88d0112f71c 2024-04-30T10:05:49.184 INFO:teuthology.orchestra.run.smithi026.stdout:iscsi.foo.smithi026.romfrr smithi026 error 2m ago 33m 64.5M - 3.5 e1d6a67b021e c9cab9bd03e1 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi046 *:8443,9283,8765 running (13m) 2m ago 38m 456M - 19.0.0-3357-g3a873b0c c50cf5fa633a 40603027226c 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (23m) 2m ago 40m 512M - 19.0.0-3357-g3a873b0c c50cf5fa633a e88d7265bd5b 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (9m) 2m ago 40m 62.0M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 649ef76a25d1 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi046 running (10m) 2m ago 39m 48.7M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 9bfb997f7d88 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (9m) 2m ago 39m 51.3M 2048M 19.0.0-3357-g3a873b0c c50cf5fa633a 44d33511609f 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (12m) 2m ago 35m 16.3M - 1.5.0 0da6a335fe13 449290d188b2 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi046 *:9100 running (12m) 2m ago 34m 16.8M - 1.5.0 0da6a335fe13 0a374c1f04f7 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (7m) 2m ago 38m 55.6M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 860b97e5bf6e 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (6m) 2m ago 38m 54.2M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 79002116badc 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (5m) 2m ago 37m 48.1M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 170943dc4c2a 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (4m) 2m ago 37m 62.2M 2993M 19.0.0-3357-g3a873b0c c50cf5fa633a 8d72616b503d 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi046 running (4m) 2m ago 36m 57.2M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a 501e0c080dad 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi046 running (4m) 2m ago 36m 47.8M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b070c14531b0 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi046 running (3m) 2m ago 36m 47.4M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a ce35c1eb8525 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi046 running (3m) 2m ago 35m 53.7M 3249M 19.0.0-3357-g3a873b0c c50cf5fa633a b081d959354e 2024-04-30T10:05:49.185 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi046 *:9095 running (12m) 2m ago 35m 80.3M - 2.43.0 a07b618ecd1d eab2d2da881a 2024-04-30T10:05:49.186 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi026.igccwv smithi026 *:8000 running (2m) 2m ago 33m 77.2M - 19.0.0-3357-g3a873b0c c50cf5fa633a cc20e54b564a 2024-04-30T10:05:49.186 INFO:teuthology.orchestra.run.smithi026.stdout:rgw.foo.smithi046.wmzons smithi046 *:8000 running (2m) 2m ago 33m 73.9M - 19.0.0-3357-g3a873b0c c50cf5fa633a a868ebccfdb5 2024-04-30T10:05:49.800 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions' 2024-04-30T10:05:50.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:50 smithi026 ceph-mon[86310]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:50.443 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:50 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:50.443 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:50 smithi026 ceph-mon[86310]: from='client.54823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:05:50.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:50 smithi026 ceph-mon[80969]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:50.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:50 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:50.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:50 smithi026 ceph-mon[80969]: from='client.54823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:05:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:50 smithi046 ceph-mon[65689]: pgmap v375: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:50 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:05:50.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:50 smithi046 ceph-mon[65689]: from='client.54823 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 3 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 8 2024-04-30T10:05:51.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:05:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw": { 2024-04-30T10:05:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 2 2024-04-30T10:05:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2024-04-30T10:05:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2024-04-30T10:05:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)": 15 2024-04-30T10:05:51.975 INFO:teuthology.orchestra.run.smithi026.stdout: } 2024-04-30T10:05:51.975 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:05:52.297 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:52 smithi026 ceph-mon[86310]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:52.297 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:52 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2302006054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:05:52.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:52 smithi026 ceph-mon[80969]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:52.297 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:52 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2302006054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:05:52.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:52 smithi046 ceph-mon[65689]: pgmap v376: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:52.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:52 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2302006054' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:05:52.637 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch upgrade status' 2024-04-30T10:05:54.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:54 smithi046 ceph-mon[65689]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:54.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:54 smithi026 ceph-mon[86310]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:54.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:54 smithi026 ceph-mon[80969]: pgmap v377: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d", 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "in_progress": true, 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "which": "Upgrading all daemon types on all hosts", 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "services_complete": [ 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr", 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw", 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "mon", 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "osd" 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "progress": "15/21 daemons upgraded", 2024-04-30T10:05:54.692 INFO:teuthology.orchestra.run.smithi026.stdout: "message": "Error: UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed.", 2024-04-30T10:05:54.693 INFO:teuthology.orchestra.run.smithi026.stdout: "is_paused": true 2024-04-30T10:05:54.693 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:05:55.379 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph health detail' 2024-04-30T10:05:56.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:56 smithi026 ceph-mon[86310]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:56.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:56 smithi026 ceph-mon[86310]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:05:56.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:56 smithi026 ceph-mon[80969]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:56.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:56 smithi026 ceph-mon[80969]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:05:56.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:56 smithi046 ceph-mon[65689]: pgmap v378: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 0 op/s 2024-04-30T10:05:56.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:56 smithi046 ceph-mon[65689]: from='client.44904 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:05:56.693 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:05:56 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:05:56] "GET /metrics HTTP/1.1" 200 38404 "" "Prometheus/2.43.0" 2024-04-30T10:05:57.464 INFO:teuthology.orchestra.run.smithi026.stdout:HEALTH_WARN 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state']; Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed. 2024-04-30T10:05:57.464 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state'] 2024-04-30T10:05:57.464 INFO:teuthology.orchestra.run.smithi026.stdout: daemon iscsi.foo.smithi026.romfrr on smithi026 is in error state 2024-04-30T10:05:57.464 INFO:teuthology.orchestra.run.smithi026.stdout:[WRN] UPGRADE_REDEPLOY_DAEMON: Upgrading daemon iscsi.foo.smithi026.romfrr on host smithi026 failed. 2024-04-30T10:05:57.464 INFO:teuthology.orchestra.run.smithi026.stdout: Upgrade daemon: iscsi.foo.smithi026.romfrr: cephadm exited with an error code: 1, stderr: Redeploy daemon iscsi.foo.smithi026.romfrr ... 2024-04-30T10:05:57.464 INFO:teuthology.orchestra.run.smithi026.stdout:Creating ceph-iscsi config... 2024-04-30T10:05:57.464 INFO:teuthology.orchestra.run.smithi026.stdout:Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/iscsi-gateway.cfg 2024-04-30T10:05:57.464 INFO:teuthology.orchestra.run.smithi026.stdout:Write file: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/iscsi.foo.smithi026.romfrr/tcmu-runner-entrypoint.sh 2024-04-30T10:05:57.464 INFO:teuthology.orchestra.run.smithi026.stdout:Failed to trim old cgroups /sys/fs/cgroup/system.slice/system-ceph\x2d703f4aea\x2d06d3\x2d11ef\x2dbc93\x2dc7b262605968.slice/ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl start ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout:Traceback (most recent call last): 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 197, in _run_module_as_main 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: return _run_code(code, main_globals, None, 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: File "/usr/lib64/python3.9/runpy.py", line 87, in _run_code 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: exec(code, run_globals) 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5642, in 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 5630, in main 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3117, in command_deploy_from 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3152, in _common_deploy 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 3172, in _deploy_daemon_container 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1087, in deploy_daemon 2024-04-30T10:05:57.465 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/__main__.py", line 1223, in deploy_daemon_units 2024-04-30T10:05:57.466 INFO:teuthology.orchestra.run.smithi026.stdout: File "/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/cephadm.d84c622a82b85bc5e361213bd3580551b7bf304674a69a5cbfef915f79b23774/cephadmlib/call_wrappers.py", line 307, in call_throws 2024-04-30T10:05:57.466 INFO:teuthology.orchestra.run.smithi026.stdout:RuntimeError: Failed command: systemctl start ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr: Job for ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service failed because the control process exited with error code. 2024-04-30T10:05:57.466 INFO:teuthology.orchestra.run.smithi026.stdout:See "systemctl status ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" and "journalctl -xeu ceph-703f4aea-06d3-11ef-bc93-c7b262605968@iscsi.foo.smithi026.romfrr.service" for details. 2024-04-30T10:05:58.110 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.overall | length == 1'"'"'' 2024-04-30T10:05:58.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:58 smithi026 ceph-mon[80969]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:58.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:05:58 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2047213390' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:05:58.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:58 smithi026 ceph-mon[86310]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:58.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:05:58 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2047213390' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:05:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:58 smithi046 ceph-mon[65689]: pgmap v379: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:05:58.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:05:58 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2047213390' entity='client.admin' cmd=[{"prefix": "health", "detail": "detail"}]: dispatch 2024-04-30T10:06:00.229 INFO:teuthology.orchestra.run.smithi026.stdout:true 2024-04-30T10:06:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:00 smithi046 ceph-mon[65689]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:00.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:00 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:06:00.498 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:00 smithi026 ceph-mon[86310]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:00.498 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:00 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:06:00.498 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:00 smithi026 ceph-mon[80969]: pgmap v380: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:00.498 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:00 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:06:00.934 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph versions | jq -e '"'"'.overall | keys'"'"' | grep $sha1' 2024-04-30T10:06:01.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:01 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/1563525135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:06:01.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:01 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:06:01.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:01 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/1563525135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:06:01.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:01 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:06:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:01 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/1563525135' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:06:01.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:01 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:06:02.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:02 smithi026 ceph-mon[86310]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:02.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:02 smithi026 ceph-mon[80969]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:02.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:02 smithi046 ceph-mon[65689]: pgmap v381: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:03.210 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 19.0.0-3357-g3a873b0c (3a873b0ce83192ee05b56734dad1076a7a94ecc7) squid (dev)" 2024-04-30T10:06:03.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:03 smithi046 ceph-mon[65689]: from='client.? 172.21.15.26:0/2065772788' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:06:03.523 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:03 smithi026 ceph-mon[86310]: from='client.? 172.21.15.26:0/2065772788' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:06:03.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:03 smithi026 ceph-mon[80969]: from='client.? 172.21.15.26:0/2065772788' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-30T10:06:03.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -e sha1=3a873b0ce83192ee05b56734dad1076a7a94ecc7 -- bash -c 'ceph orch ls | grep '"'"'^osd '"'"'' 2024-04-30T10:06:04.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:04 smithi026 ceph-mon[86310]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:06:04.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:04 smithi026 ceph-mon[80969]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:06:04.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:04 smithi046 ceph-mon[65689]: pgmap v382: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:06:05.869 INFO:teuthology.orchestra.run.smithi026.stdout:osd 8 2m ago - 2024-04-30T10:06:06.439 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-30T10:06:06.452 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2024-04-30T10:06:06.452 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls' 2024-04-30T10:06:06.453 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:06 smithi026 ceph-mon[80969]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:06.453 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:06 smithi026 ceph-mon[86310]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:06.453 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:06 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:06:06] "GET /metrics HTTP/1.1" 200 38407 "" "Prometheus/2.43.0" 2024-04-30T10:06:06.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:06 smithi046 ceph-mon[65689]: pgmap v383: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:07.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:07 smithi026 ceph-mon[86310]: from='client.54859 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:07.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:07 smithi026 ceph-mon[80969]: from='client.54859 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:07.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:07 smithi046 ceph-mon[65689]: from='client.54859 -' entity='client.admin' cmd=[{"prefix": "orch ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:08.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:08 smithi026 ceph-mon[86310]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:06:08.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:08 smithi026 ceph-mon[80969]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:06:08.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:08 smithi046 ceph-mon[65689]: pgmap v384: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:06:09.121 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2024-04-30T10:06:09.121 INFO:teuthology.orchestra.run.smithi026.stdout: "image": "quay.io/ceph/ceph", 2024-04-30T10:06:09.121 INFO:teuthology.orchestra.run.smithi026.stdout: "registry": "quay.io", 2024-04-30T10:06:09.121 INFO:teuthology.orchestra.run.smithi026.stdout: "bare_image": "ceph/ceph", 2024-04-30T10:06:09.121 INFO:teuthology.orchestra.run.smithi026.stdout: "versions": [] 2024-04-30T10:06:09.121 INFO:teuthology.orchestra.run.smithi026.stdout:} 2024-04-30T10:06:09.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:09 smithi026 ceph-mon[86310]: from='client.54865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:09.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:09 smithi026 ceph-mon[80969]: from='client.54865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:09.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:09 smithi046 ceph-mon[65689]: from='client.54865 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:09.762 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0' 2024-04-30T10:06:10.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:10 smithi026 ceph-mon[86310]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:10.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:10 smithi026 ceph-mon[86310]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:06:10.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:10 smithi026 ceph-mon[80969]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:10.337 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:10 smithi026 ceph-mon[80969]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:06:10.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:10 smithi046 ceph-mon[65689]: pgmap v385: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:10.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:10 smithi046 ceph-mon[65689]: from='client.44817 -' entity='client.iscsi.foo.smithi026.romfrr' cmd=[{"prefix": "service status", "format": "json"}]: dispatch 2024-04-30T10:06:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: "16.2.0", 2024-04-30T10:06:12.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:12 smithi026 ceph-mon[86310]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:12.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:12 smithi026 ceph-mon[80969]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:12.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:12 smithi046 ceph-mon[65689]: pgmap v386: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:13.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 -- bash -c 'ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2' 2024-04-30T10:06:13.410 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:13 smithi026 ceph-mon[86310]: from='client.54871 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:13.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:13 smithi026 ceph-mon[80969]: from='client.54871 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:13.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:13 smithi046 ceph-mon[65689]: from='client.54871 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "show_all_versions": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:14.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:14 smithi026 ceph-mon[80969]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:06:14.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:14 smithi026 ceph-mon[86310]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:06:14.734 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:14 smithi046 ceph-mon[65689]: pgmap v387: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1 op/s 2024-04-30T10:06:15.731 INFO:teuthology.orchestra.run.smithi026.stdout: "v16.2.2", 2024-04-30T10:06:15.731 INFO:teuthology.orchestra.run.smithi026.stdout: "v16.2.2-20210505", 2024-04-30T10:06:16.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-mon[86310]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:16.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-mon[86310]: from='client.44946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:16.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-mon[86310]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:06:16.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-mon[80969]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:16.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-mon[80969]: from='client.44946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:16.290 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-mon[80969]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:06:16.319 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-30T10:06:16.328 INFO:tasks.cephadm:Teardown begin 2024-04-30T10:06:16.329 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-30T10:06:16.360 DEBUG:teuthology.orchestra.run.smithi046:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-30T10:06:16.393 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-30T10:06:16.393 DEBUG:teuthology.orchestra.run.smithi026:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-30T10:06:16.417 DEBUG:teuthology.orchestra.run.smithi046:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-30T10:06:16.450 INFO:tasks.cephadm:Stopping all daemons... 2024-04-30T10:06:16.450 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-30T10:06:16.450 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.a 2024-04-30T10:06:16.538 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:16 smithi046 ceph-mon[65689]: pgmap v388: 161 pgs: 161 active+clean; 585 KiB data, 270 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 0 op/s 2024-04-30T10:06:16.538 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:16 smithi046 ceph-mon[65689]: from='client.44946 -' entity='client.admin' cmd=[{"prefix": "orch upgrade ls", "image": "quay.io/ceph/ceph", "tags": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-30T10:06:16.538 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:16 smithi046 ceph-mon[65689]: from='mgr.25696 172.21.15.26:0/709488636' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-30T10:06:16.696 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: ::ffff:172.21.15.46 - - [30/Apr/2024:10:06:16] "GET /metrics HTTP/1.1" 200 38405 "" "Prometheus/2.43.0" 2024-04-30T10:06:16.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:16 smithi026 systemd[1]: Stopping Ceph mon.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:17.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a[80965]: 2024-04-30T10:06:16.784+0000 7fe10abc0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:06:17.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a[80965]: 2024-04-30T10:06:16.784+0000 7fe10abc0700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-30T10:06:17.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:16 smithi026 podman[127609]: 2024-04-30 10:06:16.845730808 +0000 UTC m=+0.191867168 container died 649ef76a25d18a8abaa771a4b8d435bf3bbac0c5fa1da504be5a45589e6f4f56 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:06:17.169 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:10:06:16] ENGINE Bus STOPPING 2024-04-30T10:06:17.169 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:10:06:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-30T10:06:17.169 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:10:06:16] ENGINE Bus STOPPED 2024-04-30T10:06:17.169 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:10:06:16] ENGINE Bus STARTING 2024-04-30T10:06:17.169 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: CherryPy Checker: 2024-04-30T10:06:17.169 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: The Application mounted at '' has an empty config. 2024-04-30T10:06:17.170 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T10:06:17.170 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:10:06:16] ENGINE Serving on http://:::9283 2024-04-30T10:06:17.170 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:16 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: [30/Apr/2024:10:06:16] ENGINE Bus STARTED 2024-04-30T10:06:17.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:17 smithi026 podman[127609]: 2024-04-30 10:06:17.276417303 +0000 UTC m=+0.622553663 container cleanup 649ef76a25d18a8abaa771a4b8d435bf3bbac0c5fa1da504be5a45589e6f4f56 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:06:17.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:17 smithi026 bash[127609]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a 2024-04-30T10:06:17.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Apr 30 10:06:17 smithi026 podman[127634]: 2024-04-30 10:06:17.560932109 +0000 UTC m=+0.718967594 container remove 649ef76a25d18a8abaa771a4b8d435bf3bbac0c5fa1da504be5a45589e6f4f56 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-a, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-04-30T10:06:17.854 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.a.service' 2024-04-30T10:06:17.903 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:06:17.903 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-30T10:06:17.904 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-30T10:06:17.904 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.c 2024-04-30T10:06:18.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:17 smithi026 systemd[1]: Stopping Ceph mon.c for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:18 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c[86306]: 2024-04-30T10:06:18.180+0000 7f7eff8a4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:06:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:18 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c[86306]: 2024-04-30T10:06:18.180+0000 7f7eff8a4700 -1 mon.c@1(peon) e4 *** Got Signal Terminated *** 2024-04-30T10:06:18.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:18 smithi026 podman[127741]: 2024-04-30 10:06:18.303877704 +0000 UTC m=+0.216704770 container died 44d33511609ff073ddb66ef9874b7cdea20a7cc2065b5a0819f2683f20375938 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:06:18.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:18 smithi026 podman[127741]: 2024-04-30 10:06:18.72263032 +0000 UTC m=+0.635457421 container cleanup 44d33511609ff073ddb66ef9874b7cdea20a7cc2065b5a0819f2683f20375938 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True) 2024-04-30T10:06:18.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Apr 30 10:06:18 smithi026 bash[127741]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-c 2024-04-30T10:06:19.157 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.c.service' 2024-04-30T10:06:19.192 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:06:19.192 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-30T10:06:19.193 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-30T10:06:19.193 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.b 2024-04-30T10:06:19.484 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:19 smithi046 systemd[1]: Stopping Ceph mon.b for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b[65685]: 2024-04-30T10:06:19.523+0000 7f2f1e762700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:06:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b[65685]: 2024-04-30T10:06:19.523+0000 7f2f1e762700 -1 mon.b@2(peon) e4 *** Got Signal Terminated *** 2024-04-30T10:06:19.984 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:19 smithi046 podman[100241]: 2024-04-30 10:06:19.768467718 +0000 UTC m=+0.367185625 container died 9bfb997f7d880f28d1c60241527360159d2b018853689f4034a8e63c04290d99 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:06:20.695 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:20 smithi046 podman[100241]: 2024-04-30 10:06:20.352862104 +0000 UTC m=+0.951580037 container cleanup 9bfb997f7d880f28d1c60241527360159d2b018853689f4034a8e63c04290d99 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-30T10:06:20.695 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:20 smithi046 bash[100241]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b 2024-04-30T10:06:20.695 INFO:journalctl@ceph.mon.b.smithi046.stdout:Apr 30 10:06:20 smithi046 podman[100255]: 2024-04-30 10:06:20.587464812 +0000 UTC m=+0.820372078 container remove 9bfb997f7d880f28d1c60241527360159d2b018853689f4034a8e63c04290d99 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mon-b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240424, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0) 2024-04-30T10:06:20.717 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mon.b.service' 2024-04-30T10:06:20.886 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:06:20.887 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-30T10:06:20.887 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-30T10:06:20.887 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.y 2024-04-30T10:06:21.194 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:20 smithi026 systemd[1]: Stopping Ceph mgr.y for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:21.446 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T10:06:21.192+0000 7f3ea714b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:06:21.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:21 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y[60494]: 2024-04-30T10:06:21.192+0000 7f3ea714b700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-04-30T10:06:21.447 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:21 smithi026 podman[127864]: 2024-04-30 10:06:21.225223777 +0000 UTC m=+0.146479046 container died e88d7265bd5b4866a015ab9005b08085602ae231f5297df4d8d43efaefa9b4b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, ceph=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-04-30T10:06:21.920 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:21 smithi026 podman[127864]: 2024-04-30 10:06:21.610625555 +0000 UTC m=+0.531880823 container cleanup e88d7265bd5b4866a015ab9005b08085602ae231f5297df4d8d43efaefa9b4b4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:06:21.921 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Apr 30 10:06:21 smithi026 bash[127864]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-y 2024-04-30T10:06:22.101 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.y.service' 2024-04-30T10:06:22.136 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:06:22.136 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-30T10:06:22.136 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-30T10:06:22.136 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.x 2024-04-30T10:06:22.486 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 10:06:22 smithi046 systemd[1]: Stopping Ceph mgr.x for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:22.917 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 10:06:22 smithi046 podman[100362]: 2024-04-30 10:06:22.630219388 +0000 UTC m=+0.301353631 container died 40603027226c3be2bca8436165050e9d298b31430713b09ce0e6f4b26eb229fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:06:23.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 10:06:23 smithi046 podman[100362]: 2024-04-30 10:06:23.090482084 +0000 UTC m=+0.761616363 container cleanup 40603027226c3be2bca8436165050e9d298b31430713b09ce0e6f4b26eb229fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux ) 2024-04-30T10:06:23.234 INFO:journalctl@ceph.mgr.x.smithi046.stdout:Apr 30 10:06:23 smithi046 bash[100362]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-mgr-x 2024-04-30T10:06:23.485 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@mgr.x.service' 2024-04-30T10:06:23.521 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:06:23.522 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-30T10:06:23.522 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-30T10:06:23.522 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.0 2024-04-30T10:06:23.815 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:23 smithi026 systemd[1]: Stopping Ceph osd.0 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:24.196 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[93660]: 2024-04-30T10:06:23.814+0000 7f276809e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:06:24.196 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[93660]: 2024-04-30T10:06:23.814+0000 7f276809e700 -1 osd.0 160 *** Got signal Terminated *** 2024-04-30T10:06:24.196 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:23 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0[93660]: 2024-04-30T10:06:23.814+0000 7f276809e700 -1 osd.0 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:06:29.441 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:29 smithi026 podman[127985]: 2024-04-30 10:06:29.172979153 +0000 UTC m=+5.456139010 container died 860b97e5bf6e899e2c1a36a5d0034d00d963340fed4820f10bd2d989fc715d60 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-30T10:06:29.696 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:29 smithi026 podman[127985]: 2024-04-30 10:06:29.531224168 +0000 UTC m=+5.814384036 container cleanup 860b97e5bf6e899e2c1a36a5d0034d00d963340fed4820f10bd2d989fc715d60 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-04-30T10:06:29.697 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:29 smithi026 bash[127985]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0 2024-04-30T10:06:30.162 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:29 smithi026 podman[128003]: 2024-04-30 10:06:29.833042374 +0000 UTC m=+0.661535795 container remove 860b97e5bf6e899e2c1a36a5d0034d00d963340fed4820f10bd2d989fc715d60 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:06:30.446 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:30 smithi026 podman[128095]: 2024-04-30 10:06:30.064570627 +0000 UTC m=+0.018768041 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:06:30.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:30 smithi026 podman[128095]: 2024-04-30 10:06:30.167225937 +0000 UTC m=+0.121423403 container create 5c5c17d4bb90aa96d767142a0371ac7e3e39712766ce568bc98296ecfd133494 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD) 2024-04-30T10:06:30.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:30 smithi026 podman[128095]: 2024-04-30 10:06:30.370147451 +0000 UTC m=+0.324344871 container init 5c5c17d4bb90aa96d767142a0371ac7e3e39712766ce568bc98296ecfd133494 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424) 2024-04-30T10:06:30.447 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:30 smithi026 podman[128095]: 2024-04-30 10:06:30.373722992 +0000 UTC m=+0.327920399 container start 5c5c17d4bb90aa96d767142a0371ac7e3e39712766ce568bc98296ecfd133494 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:06:30.947 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:30 smithi026 podman[128095]: 2024-04-30 10:06:30.452901266 +0000 UTC m=+0.407098693 container attach 5c5c17d4bb90aa96d767142a0371ac7e3e39712766ce568bc98296ecfd133494 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-30T10:06:30.947 INFO:journalctl@ceph.osd.0.smithi026.stdout:Apr 30 10:06:30 smithi026 podman[128095]: 2024-04-30 10:06:30.681805827 +0000 UTC m=+0.636003222 container died 5c5c17d4bb90aa96d767142a0371ac7e3e39712766ce568bc98296ecfd133494 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-0-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:06:31.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.0.service' 2024-04-30T10:06:31.358 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:06:31.358 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-30T10:06:31.358 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-30T10:06:31.358 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.1 2024-04-30T10:06:31.836 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:31 smithi026 systemd[1]: Stopping Ceph osd.1 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:32.196 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[98419]: 2024-04-30T10:06:31.835+0000 7f8cd5f45700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:06:32.196 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[98419]: 2024-04-30T10:06:31.835+0000 7f8cd5f45700 -1 osd.1 160 *** Got signal Terminated *** 2024-04-30T10:06:32.196 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1[98419]: 2024-04-30T10:06:31.835+0000 7f8cd5f45700 -1 osd.1 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:06:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:06:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:06:32.616Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-30T10:06:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:06:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:06:32.617Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-30T10:06:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:06:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:06:32.618Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-30T10:06:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:06:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:06:32.620Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-30T10:06:32.985 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:06:32 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:06:32.620Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.26:8765: connect: connection refused" 2024-04-30T10:06:34.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:06:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:06:34.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T10:06:34.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:06:34 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:06:34.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T10:06:37.576 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:37 smithi026 podman[128204]: 2024-04-30 10:06:37.236835141 +0000 UTC m=+5.513596554 container died 79002116badcecb2f5b65e41e21979c5ec18e5bec7b70e4858cb0d842e5a47a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-04-30T10:06:37.577 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:37 smithi026 podman[128204]: 2024-04-30 10:06:37.576190103 +0000 UTC m=+5.852951511 container cleanup 79002116badcecb2f5b65e41e21979c5ec18e5bec7b70e4858cb0d842e5a47a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:06:37.946 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:37 smithi026 bash[128204]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1 2024-04-30T10:06:37.947 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:37 smithi026 podman[128230]: 2024-04-30 10:06:37.785768092 +0000 UTC m=+0.550393344 container remove 79002116badcecb2f5b65e41e21979c5ec18e5bec7b70e4858cb0d842e5a47a2 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-04-30T10:06:38.506 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:38 smithi026 podman[128324]: 2024-04-30 10:06:38.104500505 +0000 UTC m=+0.018659045 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:06:38.506 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:38 smithi026 podman[128324]: 2024-04-30 10:06:38.219941377 +0000 UTC m=+0.134099907 container create fb857b9a07a650f18f061dd625828a746c3e0f1abf498d473fa7b32ab9facba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-04-30T10:06:38.506 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:38 smithi026 podman[128324]: 2024-04-30 10:06:38.440015173 +0000 UTC m=+0.354173702 container init fb857b9a07a650f18f061dd625828a746c3e0f1abf498d473fa7b32ab9facba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-04-30T10:06:38.506 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:38 smithi026 podman[128324]: 2024-04-30 10:06:38.443254873 +0000 UTC m=+0.357413409 container start fb857b9a07a650f18f061dd625828a746c3e0f1abf498d473fa7b32ab9facba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:06:38.506 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:38 smithi026 podman[128324]: 2024-04-30 10:06:38.50567475 +0000 UTC m=+0.419833295 container attach fb857b9a07a650f18f061dd625828a746c3e0f1abf498d473fa7b32ab9facba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-04-30T10:06:39.051 INFO:journalctl@ceph.osd.1.smithi026.stdout:Apr 30 10:06:38 smithi026 podman[128324]: 2024-04-30 10:06:38.757369999 +0000 UTC m=+0.671528534 container died fb857b9a07a650f18f061dd625828a746c3e0f1abf498d473fa7b32ab9facba1 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-1-deactivate, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:06:39.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.1.service' 2024-04-30T10:06:39.478 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:06:39.478 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-30T10:06:39.479 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-30T10:06:39.479 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.2 2024-04-30T10:06:39.697 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:39 smithi026 systemd[1]: Stopping Ceph osd.2 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:40.196 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:39 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[105144]: 2024-04-30T10:06:39.812+0000 7f3dbb734700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:06:40.197 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:39 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[105144]: 2024-04-30T10:06:39.812+0000 7f3dbb734700 -1 osd.2 160 *** Got signal Terminated *** 2024-04-30T10:06:40.197 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:39 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2[105144]: 2024-04-30T10:06:39.812+0000 7f3dbb734700 -1 osd.2 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:06:44.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:06:44 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:06:44.271Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T10:06:44.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:06:44 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:06:44.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T10:06:44.698 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:06:44 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:06:44.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T10:06:45.446 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:45 smithi026 podman[128428]: 2024-04-30 10:06:45.179242448 +0000 UTC m=+5.475828600 container died 170943dc4c2ad3990fc520cce24dd16d4cc324de7da95df463c7d33cc8aed065 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-30T10:06:45.946 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:45 smithi026 podman[128428]: 2024-04-30 10:06:45.486998011 +0000 UTC m=+5.783584161 container cleanup 170943dc4c2ad3990fc520cce24dd16d4cc324de7da95df463c7d33cc8aed065 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True) 2024-04-30T10:06:45.947 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:45 smithi026 bash[128428]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2 2024-04-30T10:06:45.947 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:45 smithi026 podman[128446]: 2024-04-30 10:06:45.696453237 +0000 UTC m=+0.513972709 container remove 170943dc4c2ad3990fc520cce24dd16d4cc324de7da95df463c7d33cc8aed065 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-30T10:06:46.333 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:46 smithi026 podman[128537]: 2024-04-30 10:06:45.946384475 +0000 UTC m=+0.018411943 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:06:46.333 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:46 smithi026 podman[128537]: 2024-04-30 10:06:46.047443883 +0000 UTC m=+0.119471315 container create 18e36fd5ffe6d3481f25019a399be2bdfa7d18a4a813d1f5d60a911f33f01448 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424) 2024-04-30T10:06:46.333 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:46 smithi026 podman[128537]: 2024-04-30 10:06:46.258772017 +0000 UTC m=+0.330799463 container init 18e36fd5ffe6d3481f25019a399be2bdfa7d18a4a813d1f5d60a911f33f01448 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True) 2024-04-30T10:06:46.333 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:46 smithi026 podman[128537]: 2024-04-30 10:06:46.262192898 +0000 UTC m=+0.334220336 container start 18e36fd5ffe6d3481f25019a399be2bdfa7d18a4a813d1f5d60a911f33f01448 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240424, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-04-30T10:06:46.333 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:46 smithi026 podman[128537]: 2024-04-30 10:06:46.333053224 +0000 UTC m=+0.405080674 container attach 18e36fd5ffe6d3481f25019a399be2bdfa7d18a4a813d1f5d60a911f33f01448 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-04-30T10:06:46.696 INFO:journalctl@ceph.osd.2.smithi026.stdout:Apr 30 10:06:46 smithi026 podman[128537]: 2024-04-30 10:06:46.560516642 +0000 UTC m=+0.632544088 container died 18e36fd5ffe6d3481f25019a399be2bdfa7d18a4a813d1f5d60a911f33f01448 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:06:47.123 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.2.service' 2024-04-30T10:06:47.162 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:06:47.162 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-30T10:06:47.162 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-30T10:06:47.162 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.3 2024-04-30T10:06:47.447 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:47 smithi026 systemd[1]: Stopping Ceph osd.3 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:47.946 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:47 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[111437]: 2024-04-30T10:06:47.505+0000 7f8ec6f4a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:06:47.947 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:47 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[111437]: 2024-04-30T10:06:47.505+0000 7f8ec6f4a700 -1 osd.3 160 *** Got signal Terminated *** 2024-04-30T10:06:47.947 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:47 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[111437]: 2024-04-30T10:06:47.505+0000 7f8ec6f4a700 -1 osd.3 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:06:50.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:50 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:50.200+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:50.746 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:50 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:50.533+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:51.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:51 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:51.161+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:51.953 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:51 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:51.485+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:51.953 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:51 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:51.524+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:06:52.197 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:51 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3[111437]: 2024-04-30T10:06:51.739+0000 7f8ebf536700 -1 osd.3 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.620945+0000 front 2024-04-30T10:06:28.620982+0000 (oldest deadline 2024-04-30T10:06:50.821657+0000) 2024-04-30T10:06:52.234 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:52.112+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:52.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:51 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:51.950+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:06:52.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:52.509+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:52.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:52 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:52.528+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:06:53.190 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:52 smithi026 podman[128641]: 2024-04-30 10:06:52.925561944 +0000 UTC m=+5.541070699 container died 8d72616b503d7ca10188afe6c9148b810187e1d84c2f3481884dedbe357ce35c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2) 2024-04-30T10:06:53.446 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:53 smithi026 podman[128641]: 2024-04-30 10:06:53.272412244 +0000 UTC m=+5.887920991 container cleanup 8d72616b503d7ca10188afe6c9148b810187e1d84c2f3481884dedbe357ce35c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-04-30T10:06:53.447 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:53 smithi026 bash[128641]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3 2024-04-30T10:06:53.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:53.118+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:53.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:52.999+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:06:53.902 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:53 smithi026 podman[128659]: 2024-04-30 10:06:53.456955804 +0000 UTC m=+0.525210420 container remove 8d72616b503d7ca10188afe6c9148b810187e1d84c2f3481884dedbe357ce35c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:06:53.954 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:53.547+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:53.955 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:53.517+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:06:54.196 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:53 smithi026 podman[128750]: 2024-04-30 10:06:53.805012277 +0000 UTC m=+0.018691649 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:06:54.197 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:53 smithi026 podman[128750]: 2024-04-30 10:06:53.924320435 +0000 UTC m=+0.137999835 container create 746224c3479f276fa89357aab3b6cc642953e8e3eb1e7d8fea1c6cec1d33a5fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-04-30T10:06:54.197 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:54 smithi026 podman[128750]: 2024-04-30 10:06:54.126940936 +0000 UTC m=+0.340620295 container init 746224c3479f276fa89357aab3b6cc642953e8e3eb1e7d8fea1c6cec1d33a5fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:06:54.197 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:54 smithi026 podman[128750]: 2024-04-30 10:06:54.130002643 +0000 UTC m=+0.343682002 container start 746224c3479f276fa89357aab3b6cc642953e8e3eb1e7d8fea1c6cec1d33a5fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-04-30T10:06:54.234 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:54 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:54.104+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:54.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:53 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:53.952+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:06:54.696 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:54 smithi026 podman[128750]: 2024-04-30 10:06:54.218403475 +0000 UTC m=+0.432082845 container attach 746224c3479f276fa89357aab3b6cc642953e8e3eb1e7d8fea1c6cec1d33a5fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-04-30T10:06:54.697 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:54 smithi026 podman[128750]: 2024-04-30 10:06:54.441739594 +0000 UTC m=+0.655418962 container died 746224c3479f276fa89357aab3b6cc642953e8e3eb1e7d8fea1c6cec1d33a5fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-04-30T10:06:54.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:06:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:06:54.272Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T10:06:54.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:06:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:06:54.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T10:06:54.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:06:54 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:06:54.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T10:06:54.958 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:54 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:54.585+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:54.959 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:54 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:54.566+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:06:54.997 INFO:journalctl@ceph.osd.3.smithi026.stdout:Apr 30 10:06:54 smithi026 podman[128776]: 2024-04-30 10:06:54.98043829 +0000 UTC m=+0.533529421 container remove 746224c3479f276fa89357aab3b6cc642953e8e3eb1e7d8fea1c6cec1d33a5fa (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240424) 2024-04-30T10:06:55.002 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.3.service' 2024-04-30T10:06:55.043 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:06:55.043 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-30T10:06:55.044 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-30T10:06:55.044 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.4 2024-04-30T10:06:55.234 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:55 smithi046 systemd[1]: Stopping Ceph osd.4 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:06:55.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:54 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:54.957+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:06:55.234 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:55.127+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:55.734 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:55.541+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:06:55.734 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:55.359+0000 7facd83e4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:06:55.734 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:55.359+0000 7facd83e4700 -1 osd.4 160 *** Got signal Terminated *** 2024-04-30T10:06:55.734 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:55.359+0000 7facd83e4700 -1 osd.4 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:06:55.734 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:55 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:55.566+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:56.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:56 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:56.157+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:56.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:56 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:56.003+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:06:56.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:56 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:56.583+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:56.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:56 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:56.539+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:06:57.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:57.185+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:57.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:57.037+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:06:57.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:57.577+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:57.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:57 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:57.582+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:06:58.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:58.196+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:58.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:58.016+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:06:58.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:58.016+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:06:58.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:58.572+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:58.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:58.581+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:06:58.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:58.581+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:06:59.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:06:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:06:59.187+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:06:59.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:59.055+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:06:59.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:06:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:06:59.055+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:06:59.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:59.612+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:28.378305+0000 front 2024-04-30T10:06:28.378337+0000 (oldest deadline 2024-04-30T10:06:50.078405+0000) 2024-04-30T10:06:59.984 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:06:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4[76444]: 2024-04-30T10:06:59.612+0000 7facd09d0700 -1 osd.4 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:36.380369+0000 front 2024-04-30T10:06:36.380357+0000 (oldest deadline 2024-04-30T10:06:59.280241+0000) 2024-04-30T10:06:59.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:59.570+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:06:59.985 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:06:59 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:06:59.570+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:07:00.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:00.237+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:00.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:00.237+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:00.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:00.076+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:00.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:00.076+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:00.839 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:00.546+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:07:00.840 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:00 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:00.546+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:07:01.102 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:00 smithi046 podman[100499]: 2024-04-30 10:07:00.839039593 +0000 UTC m=+5.586625211 container died 501e0c080dadf56946460eb5ed4d00fc5bad879e3a1acbe48b28c5474c86a3b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True) 2024-04-30T10:07:01.484 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:01 smithi046 podman[100499]: 2024-04-30 10:07:01.302321175 +0000 UTC m=+6.049906836 container cleanup 501e0c080dadf56946460eb5ed4d00fc5bad879e3a1acbe48b28c5474c86a3b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4, GIT_CLEAN=True, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-04-30T10:07:01.484 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:01 smithi046 bash[100499]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4 2024-04-30T10:07:01.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:01 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:01.274+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:01.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:01 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:01.274+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:01.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:01 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:01.100+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:01.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:01 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:01.100+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:01.930 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:01 smithi046 podman[100516]: 2024-04-30 10:07:01.591090988 +0000 UTC m=+0.749737838 container remove 501e0c080dadf56946460eb5ed4d00fc5bad879e3a1acbe48b28c5474c86a3b3 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:07:01.930 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:01 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:01.526+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:07:01.930 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:01 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:01.526+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:07:02.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:02 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:02.097+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:02.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:02 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:02.097+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:02.234 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:01 smithi046 podman[100609]: 2024-04-30 10:07:01.929759153 +0000 UTC m=+0.108747277 container create 6557c1a6f9d8c66ad512680acd4ab4221a5e6ae1543da06627b6e6e35efd2b5e (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, ceph=True, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:07:02.235 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:01 smithi046 podman[100609]: 2024-04-30 10:07:01.839462098 +0000 UTC m=+0.018450242 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:07:02.235 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:02 smithi046 podman[100609]: 2024-04-30 10:07:02.099870424 +0000 UTC m=+0.278858548 container init 6557c1a6f9d8c66ad512680acd4ab4221a5e6ae1543da06627b6e6e35efd2b5e (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, ceph=True, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:07:02.235 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:02 smithi046 podman[100609]: 2024-04-30 10:07:02.103315607 +0000 UTC m=+0.282303734 container start 6557c1a6f9d8c66ad512680acd4ab4221a5e6ae1543da06627b6e6e35efd2b5e (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:07:02.235 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:02 smithi046 podman[100609]: 2024-04-30 10:07:02.173816957 +0000 UTC m=+0.352805094 container attach 6557c1a6f9d8c66ad512680acd4ab4221a5e6ae1543da06627b6e6e35efd2b5e (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:07:02.521 INFO:journalctl@ceph.osd.4.smithi046.stdout:Apr 30 10:07:02 smithi046 podman[100609]: 2024-04-30 10:07:02.414568622 +0000 UTC m=+0.593556746 container died 6557c1a6f9d8c66ad512680acd4ab4221a5e6ae1543da06627b6e6e35efd2b5e (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-4-deactivate, GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:07:02.521 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:02 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:02.247+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:02.522 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:02 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:02.247+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:02.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:02 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:02.519+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:07:02.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:02 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:02.519+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:07:03.036 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.4.service' 2024-04-30T10:07:03.080 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:07:03.081 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-30T10:07:03.081 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-30T10:07:03.081 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.5 2024-04-30T10:07:03.284 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:03.246+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:03.284 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:03.246+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:03.284 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:03.105+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:03.284 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:03.105+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:03.285 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:03 smithi046 systemd[1]: Stopping Ceph osd.5 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:07:03.555 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:03.437+0000 7f63d9a8b700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:07:03.555 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:03.437+0000 7f63d9a8b700 -1 osd.5 160 *** Got signal Terminated *** 2024-04-30T10:07:03.555 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:03.437+0000 7f63d9a8b700 -1 osd.5 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:07:03.555 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:03.552+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:07:03.555 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:03 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:03.552+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:07:04.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:04 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:04.199+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:04.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:04 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:04.199+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:04.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:04 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:04.073+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:04.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:04 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:04.073+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:04.696 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:04.273Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T10:07:04.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:04.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T10:07:04.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:04 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:04.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T10:07:04.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:04 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:04.551+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:07:04.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:04 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:04.551+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:07:05.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:05.235+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:05.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:05.235+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:05.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:05.087+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:05.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:05.087+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:05.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:05.567+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:07:05.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:05 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:05.567+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:07:06.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:06.067+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:06.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:06.067+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:06.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:06.241+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:06.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:06.241+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:06.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:06.241+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:06.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:06.522+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:07:06.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:06 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:06.522+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:07:07.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:07.115+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:07.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:07.115+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:07.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:07.220+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:07.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:07.220+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:07.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:07.220+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:07.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:07.547+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.918238+0000 front 2024-04-30T10:06:27.918332+0000 (oldest deadline 2024-04-30T10:06:50.718680+0000) 2024-04-30T10:07:07.984 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:07 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5[81036]: 2024-04-30T10:07:07.547+0000 7f63d607f700 -1 osd.5 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:33.619860+0000 front 2024-04-30T10:06:33.619878+0000 (oldest deadline 2024-04-30T10:06:58.319543+0000) 2024-04-30T10:07:08.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:08.121+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:08.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:08.121+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:08.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:08.180+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:08.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:08.180+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:08.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:08 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:08.180+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:09.234 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:08 smithi046 podman[100715]: 2024-04-30 10:07:08.964477516 +0000 UTC m=+5.651713424 container died b070c14531b045b8698b4eba2907d9d7daed539f4e24a4a38ba894ff963e517b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-04-30T10:07:09.234 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:09 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:09.150+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:09.235 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:09 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:09.150+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:09.235 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:09 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:09.150+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:09.235 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:09 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:09.160+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:09.235 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:09 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:09.160+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:09.235 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:09 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:09.160+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:09.674 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:09 smithi046 podman[100715]: 2024-04-30 10:07:09.414146721 +0000 UTC m=+6.101382627 container cleanup b070c14531b045b8698b4eba2907d9d7daed539f4e24a4a38ba894ff963e517b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True) 2024-04-30T10:07:09.674 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:09 smithi046 bash[100715]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5 2024-04-30T10:07:09.975 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:09 smithi046 podman[100743]: 2024-04-30 10:07:09.673745357 +0000 UTC m=+0.706474438 container remove b070c14531b045b8698b4eba2907d9d7daed539f4e24a4a38ba894ff963e517b (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:07:10.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:10 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:10.179+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:10.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:10 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:10.179+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:10.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:10 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:10.179+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:10.235 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:10 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:10.197+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:10.235 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:10 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:10.197+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:10.235 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:10 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:10.197+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:10.235 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:09 smithi046 podman[100836]: 2024-04-30 10:07:09.97448037 +0000 UTC m=+0.109826186 container create a84af77cab711bddc201da6316c14fd3c009b387b5a9b0b9b68190f48bae656c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-30T10:07:10.235 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:09 smithi046 podman[100836]: 2024-04-30 10:07:09.88331005 +0000 UTC m=+0.018655894 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:07:10.235 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:10 smithi046 podman[100836]: 2024-04-30 10:07:10.169106294 +0000 UTC m=+0.304452119 container init a84af77cab711bddc201da6316c14fd3c009b387b5a9b0b9b68190f48bae656c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424) 2024-04-30T10:07:10.235 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:10 smithi046 podman[100836]: 2024-04-30 10:07:10.172545485 +0000 UTC m=+0.307891304 container start a84af77cab711bddc201da6316c14fd3c009b387b5a9b0b9b68190f48bae656c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_CLEAN=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:07:10.734 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:10 smithi046 podman[100836]: 2024-04-30 10:07:10.29362396 +0000 UTC m=+0.428969789 container attach a84af77cab711bddc201da6316c14fd3c009b387b5a9b0b9b68190f48bae656c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-04-30T10:07:10.734 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:10 smithi046 podman[100836]: 2024-04-30 10:07:10.474819219 +0000 UTC m=+0.610165027 container died a84af77cab711bddc201da6316c14fd3c009b387b5a9b0b9b68190f48bae656c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240424, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True) 2024-04-30T10:07:11.022 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:11 smithi046 podman[100862]: 2024-04-30 10:07:11.005598926 +0000 UTC m=+0.521777903 container remove a84af77cab711bddc201da6316c14fd3c009b387b5a9b0b9b68190f48bae656c (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-5-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240424, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-04-30T10:07:11.022 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:11 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.5.service: Deactivated successfully. 2024-04-30T10:07:11.022 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:11 smithi046 systemd[1]: Stopped Ceph osd.5 for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:07:11.023 INFO:journalctl@ceph.osd.5.smithi046.stdout:Apr 30 10:07:11 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.5.service: Consumed 5.126s CPU time. 2024-04-30T10:07:11.024 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.5.service' 2024-04-30T10:07:11.064 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:07:11.064 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-30T10:07:11.064 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-30T10:07:11.065 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.6 2024-04-30T10:07:11.384 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:11.162+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:11.384 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:11.162+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:11.384 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:11.162+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:11.384 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:11 smithi046 systemd[1]: Stopping Ceph osd.6 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:07:11.385 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:11.221+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:11.385 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:11.221+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:11.385 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:11.221+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:11.385 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:11.382+0000 7fd87aa81700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:07:11.385 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:11.382+0000 7fd87aa81700 -1 osd.6 160 *** Got signal Terminated *** 2024-04-30T10:07:11.385 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:11 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:11.382+0000 7fd87aa81700 -1 osd.6 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:07:12.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:12.151+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:12.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:12.151+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:12.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:12.151+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:12.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:12.270+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:12.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:12.270+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:12.485 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:12 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:12.270+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:13.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:13.319+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:13.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:13.319+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:13.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:13.319+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:13.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:13.130+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:13.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:13.130+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:13.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:13 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:13.130+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:14.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:14 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:14.277+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:14.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:14 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:14.278+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:14.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:14 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:14.278+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:14.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:14 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:14.173+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:14.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:14 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:14.173+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:14.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:14 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:14.173+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:14.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:14 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:14.173+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 2 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:14.608 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:14 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:14.274Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T10:07:14.608 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:14 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:14.275Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T10:07:14.608 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:14 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:14.275Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T10:07:15.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:15.242+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:15.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:15.242+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:15.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:15.242+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:15.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:15.242+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:50.632391+0000 front 2024-04-30T10:06:50.632314+0000 (oldest deadline 2024-04-30T10:07:14.732247+0000) 2024-04-30T10:07:15.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:15.187+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:15.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:15.187+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:15.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:15.187+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:15.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:15 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:15.187+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 2 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:16.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:16.257+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:27.329180+0000 front 2024-04-30T10:06:27.329296+0000 (oldest deadline 2024-04-30T10:06:49.629240+0000) 2024-04-30T10:07:16.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:16.257+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:35.331160+0000 front 2024-04-30T10:06:35.330958+0000 (oldest deadline 2024-04-30T10:06:59.430966+0000) 2024-04-30T10:07:16.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:16.257+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:39.431600+0000 front 2024-04-30T10:06:39.431500+0000 (oldest deadline 2024-04-30T10:07:05.331467+0000) 2024-04-30T10:07:16.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6[86021]: 2024-04-30T10:07:16.257+0000 7fd877075700 -1 osd.6 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:50.632391+0000 front 2024-04-30T10:06:50.632314+0000 (oldest deadline 2024-04-30T10:07:14.732247+0000) 2024-04-30T10:07:16.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:16.151+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:16.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:16.151+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:16.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:16.151+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:16.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:16 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:16.151+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 2 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:17.177 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:16 smithi046 podman[100941]: 2024-04-30 10:07:16.783130582 +0000 UTC m=+5.493564136 container died ce35c1eb8525dffd057b272472473105ce91bc83cf36d765bdfcf4da931ab120 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD) 2024-04-30T10:07:17.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:17 smithi046 podman[100941]: 2024-04-30 10:07:17.183044874 +0000 UTC m=+5.893478430 container cleanup ce35c1eb8525dffd057b272472473105ce91bc83cf36d765bdfcf4da931ab120 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS) 2024-04-30T10:07:17.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:17 smithi046 bash[100941]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6 2024-04-30T10:07:17.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:17 smithi046 podman[100959]: 2024-04-30 10:07:17.417822667 +0000 UTC m=+0.626714437 container remove ce35c1eb8525dffd057b272472473105ce91bc83cf36d765bdfcf4da931ab120 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-04-30T10:07:17.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:17.176+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:17.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:17.176+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:17.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:17.176+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:17.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:17.176+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:51.210415+0000 front 2024-04-30T10:06:51.210332+0000 (oldest deadline 2024-04-30T10:07:16.510258+0000) 2024-04-30T10:07:17.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:17 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:17.176+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 2 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:18.089 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:17 smithi046 podman[101050]: 2024-04-30 10:07:17.666949005 +0000 UTC m=+0.018475549 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:07:18.089 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:17 smithi046 podman[101050]: 2024-04-30 10:07:17.78551343 +0000 UTC m=+0.137040003 container create e906d8de60a3f427fba2d0fe02743f5e762952b9059690980d0ba6226aa62a46 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240424, CEPH_POINT_RELEASE=, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, ceph=True, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux ) 2024-04-30T10:07:18.089 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:17 smithi046 podman[101050]: 2024-04-30 10:07:17.987989007 +0000 UTC m=+0.339515537 container init e906d8de60a3f427fba2d0fe02743f5e762952b9059690980d0ba6226aa62a46 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0) 2024-04-30T10:07:18.089 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:17 smithi046 podman[101050]: 2024-04-30 10:07:17.991484301 +0000 UTC m=+0.343010839 container start e906d8de60a3f427fba2d0fe02743f5e762952b9059690980d0ba6226aa62a46 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-04-30T10:07:18.089 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:18 smithi046 podman[101050]: 2024-04-30 10:07:18.088037451 +0000 UTC m=+0.439563993 container attach e906d8de60a3f427fba2d0fe02743f5e762952b9059690980d0ba6226aa62a46 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240424, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-04-30T10:07:18.484 INFO:journalctl@ceph.osd.6.smithi046.stdout:Apr 30 10:07:18 smithi046 podman[101050]: 2024-04-30 10:07:18.297405744 +0000 UTC m=+0.648932274 container died e906d8de60a3f427fba2d0fe02743f5e762952b9059690980d0ba6226aa62a46 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-6-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, maintainer=Guillaume Abrioux , RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-04-30T10:07:18.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:18.174+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:18.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:18.174+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:18.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:18.174+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:18.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:18.174+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:51.210415+0000 front 2024-04-30T10:06:51.210332+0000 (oldest deadline 2024-04-30T10:07:16.510258+0000) 2024-04-30T10:07:18.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:18 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:18.175+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 2 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:18.996 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.6.service' 2024-04-30T10:07:19.036 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:07:19.036 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-30T10:07:19.037 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-30T10:07:19.037 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.7 2024-04-30T10:07:19.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:19 smithi046 systemd[1]: Stopping Ceph osd.7 for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:07:19.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:19.143+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:19.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:19.143+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:19.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:19.143+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:19.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:19.143+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:51.210415+0000 front 2024-04-30T10:06:51.210332+0000 (oldest deadline 2024-04-30T10:07:16.510258+0000) 2024-04-30T10:07:19.234 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:19.143+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:19.734 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:19.394+0000 7f7590f90700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-04-30T10:07:19.735 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:19.395+0000 7f7590f90700 -1 osd.7 160 *** Got signal Terminated *** 2024-04-30T10:07:19.735 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:19 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:19.395+0000 7f7590f90700 -1 osd.7 160 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-30T10:07:20.487 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:20.164+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:20.487 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:20.164+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:20.487 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:20.164+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:20.487 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:20.164+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:51.210415+0000 front 2024-04-30T10:06:51.210332+0000 (oldest deadline 2024-04-30T10:07:16.510258+0000) 2024-04-30T10:07:20.487 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:20 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:20.165+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:21.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:21.146+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:21.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:21.146+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:21.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:21.146+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:21.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:21.146+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:51.210415+0000 front 2024-04-30T10:06:51.210332+0000 (oldest deadline 2024-04-30T10:07:16.510258+0000) 2024-04-30T10:07:21.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:21 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:21.146+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:22.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:22 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:22.108+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:22.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:22 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:22.108+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:22.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:22 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:22.108+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:22.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:22 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:22.108+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:51.210415+0000 front 2024-04-30T10:06:51.210332+0000 (oldest deadline 2024-04-30T10:07:16.510258+0000) 2024-04-30T10:07:22.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:22 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:22.108+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.46:6804 osd.4 since back 2024-04-30T10:06:56.510865+0000 front 2024-04-30T10:06:56.510702+0000 (oldest deadline 2024-04-30T10:07:21.810747+0000) 2024-04-30T10:07:22.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:22 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:22.108+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:23.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:23.104+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:23.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:23.104+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:23.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:23.104+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:23.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:23.104+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:51.210415+0000 front 2024-04-30T10:06:51.210332+0000 (oldest deadline 2024-04-30T10:07:16.510258+0000) 2024-04-30T10:07:23.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:23.104+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.46:6804 osd.4 since back 2024-04-30T10:06:56.510865+0000 front 2024-04-30T10:06:56.510702+0000 (oldest deadline 2024-04-30T10:07:21.810747+0000) 2024-04-30T10:07:23.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:23 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:23.104+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 4 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:24.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:24.148+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2024-04-30T10:06:26.706507+0000 front 2024-04-30T10:06:26.706466+0000 (oldest deadline 2024-04-30T10:06:51.307172+0000) 2024-04-30T10:07:24.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:24.148+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2024-04-30T10:06:31.307623+0000 front 2024-04-30T10:06:31.307840+0000 (oldest deadline 2024-04-30T10:06:57.708267+0000) 2024-04-30T10:07:24.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:24.148+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2024-04-30T10:06:43.009468+0000 front 2024-04-30T10:06:43.009502+0000 (oldest deadline 2024-04-30T10:07:08.909361+0000) 2024-04-30T10:07:24.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:24.148+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2024-04-30T10:06:51.210415+0000 front 2024-04-30T10:06:51.210332+0000 (oldest deadline 2024-04-30T10:07:16.510258+0000) 2024-04-30T10:07:24.484 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:24.148+0000 7f758957c700 -1 osd.7 160 heartbeat_check: no reply from 172.21.15.46:6804 osd.4 since back 2024-04-30T10:06:56.510865+0000 front 2024-04-30T10:06:56.510702+0000 (oldest deadline 2024-04-30T10:07:21.810747+0000) 2024-04-30T10:07:24.485 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:24 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7[91008]: 2024-04-30T10:07:24.148+0000 7f758957c700 -1 osd.7 160 get_health_metrics reporting 6 slow ops, oldest is osd_op(client.54601.0:42 2.5 2:a0eab6e3:::realms.5a286eef-7c35-49b1-baa0-f540ab93020d.control:head [watch ping cookie 94212027827200] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160) 2024-04-30T10:07:24.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:24 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:24.275Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T10:07:24.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:24 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:24.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2024-04-30T10:07:24.697 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:24 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:24.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi046.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.46:8443: connect: connection refused" 2024-04-30T10:07:25.204 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:24 smithi046 podman[101155]: 2024-04-30 10:07:24.929351947 +0000 UTC m=+5.657932389 container died b081d959354ecd93f8e37573a90b05969960cfbfdd70e22065790e5fd9eaf319 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=) 2024-04-30T10:07:25.763 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:25 smithi046 podman[101155]: 2024-04-30 10:07:25.486358039 +0000 UTC m=+6.214938520 container cleanup b081d959354ecd93f8e37573a90b05969960cfbfdd70e22065790e5fd9eaf319 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424) 2024-04-30T10:07:25.763 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:25 smithi046 bash[101155]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7 2024-04-30T10:07:26.120 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:25 smithi046 podman[101185]: 2024-04-30 10:07:25.762909805 +0000 UTC m=+0.831687570 container remove b081d959354ecd93f8e37573a90b05969960cfbfdd70e22065790e5fd9eaf319 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, io.buildah.version=1.33.5, org.label-schema.build-date=20240424, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-04-30T10:07:26.399 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:26 smithi046 podman[101281]: 2024-04-30 10:07:26.023592361 +0000 UTC m=+0.018693054 image pull c50cf5fa633ab4fdc947f6a694b10f0ce365fb123212c28681439f9b27bff8c0 quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d 2024-04-30T10:07:26.400 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:26 smithi046 podman[101281]: 2024-04-30 10:07:26.121940471 +0000 UTC m=+0.117041196 container create 0b63539e9b18c731108c95793f36eb4d03190439798d12f1d93d2414c2ac12c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.build-date=20240424, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux ) 2024-04-30T10:07:26.400 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:26 smithi046 podman[101281]: 2024-04-30 10:07:26.324371286 +0000 UTC m=+0.319471968 container init 0b63539e9b18c731108c95793f36eb4d03190439798d12f1d93d2414c2ac12c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-04-30T10:07:26.400 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:26 smithi046 podman[101281]: 2024-04-30 10:07:26.327550934 +0000 UTC m=+0.322651622 container start 0b63539e9b18c731108c95793f36eb4d03190439798d12f1d93d2414c2ac12c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, GIT_BRANCH=HEAD, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True) 2024-04-30T10:07:26.400 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:26 smithi046 podman[101281]: 2024-04-30 10:07:26.399151971 +0000 UTC m=+0.394252653 container attach 0b63539e9b18c731108c95793f36eb4d03190439798d12f1d93d2414c2ac12c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, org.label-schema.build-date=20240424, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b, io.buildah.version=1.33.5, ceph=True, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, maintainer=Guillaume Abrioux ) 2024-04-30T10:07:26.734 INFO:journalctl@ceph.osd.7.smithi046.stdout:Apr 30 10:07:26 smithi046 podman[101281]: 2024-04-30 10:07:26.632512483 +0000 UTC m=+0.627613164 container died 0b63539e9b18c731108c95793f36eb4d03190439798d12f1d93d2414c2ac12c4 (image=quay.ceph.io/ceph-ci/ceph@sha256:9c431cd6102fdc98572343878a0f5e76680ffaef4fdd710bbca12393d613517d, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-osd-7-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240424, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=wip-adk-testing-2024-04-29-2009-3a873b0, GIT_COMMIT=318fe5928e744a8c3fe408a2f9ccaca3c31b552b) 2024-04-30T10:07:27.414 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@osd.7.service' 2024-04-30T10:07:27.461 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:07:27.461 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-30T10:07:27.461 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-30T10:07:27.461 DEBUG:teuthology.orchestra.run.smithi046:> sudo systemctl stop ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a 2024-04-30T10:07:27.734 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 systemd[1]: Stopping Ceph prometheus.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:07:28.006 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.751Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-30T10:07:28.006 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.751Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-30T10:07:28.006 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.751Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-30T10:07:28.006 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.751Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-30T10:07:28.007 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.751Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-30T10:07:28.007 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.751Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-30T10:07:28.007 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.751Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-30T10:07:28.007 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.751Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-30T10:07:28.007 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.752Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-30T10:07:28.007 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.752Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-30T10:07:28.007 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.752Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-30T10:07:28.007 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a[61855]: ts=2024-04-30T10:07:27.752Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-30T10:07:28.008 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:27 smithi046 podman[101387]: 2024-04-30 10:07:27.762054123 +0000 UTC m=+0.109937809 container died eab2d2da881a553dc2d8439f6fdfec6bad6555616417d554aba1d8e11d7961d8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T10:07:28.008 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:28 smithi046 podman[101387]: 2024-04-30 10:07:28.005467356 +0000 UTC m=+0.353351043 container cleanup eab2d2da881a553dc2d8439f6fdfec6bad6555616417d554aba1d8e11d7961d8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T10:07:28.008 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:28 smithi046 bash[101387]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a 2024-04-30T10:07:28.420 INFO:journalctl@ceph.prometheus.a.smithi046.stdout:Apr 30 10:07:28 smithi046 podman[101402]: 2024-04-30 10:07:28.18989736 +0000 UTC m=+0.426180196 container remove eab2d2da881a553dc2d8439f6fdfec6bad6555616417d554aba1d8e11d7961d8 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-04-30T10:07:28.444 DEBUG:teuthology.orchestra.run.smithi046:> sudo pkill -f 'journalctl -f -n 0 -u ceph-703f4aea-06d3-11ef-bc93-c7b262605968@prometheus.a.service' 2024-04-30T10:07:28.477 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-30T10:07:28.478 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-30T10:07:28.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-30T10:07:30.287 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 10:07:30 smithi026 systemd[1]: Stopping Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:07:30.576 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 10:07:30 smithi026 podman[129142]: 2024-04-30 10:07:30.286047822 +0000 UTC m=+0.118676105 container died 449290d188b2310e5321ad27e3df78a06cba2ba6184d15fab90666e113e335e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T10:07:30.918 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 10:07:30 smithi026 podman[129142]: 2024-04-30 10:07:30.674876839 +0000 UTC m=+0.507505114 container cleanup 449290d188b2310e5321ad27e3df78a06cba2ba6184d15fab90666e113e335e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T10:07:30.918 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 10:07:30 smithi026 bash[129142]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a 2024-04-30T10:07:31.196 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 10:07:30 smithi026 podman[129156]: 2024-04-30 10:07:30.917631298 +0000 UTC m=+0.630400357 container remove 449290d188b2310e5321ad27e3df78a06cba2ba6184d15fab90666e113e335e7 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-04-30T10:07:31.196 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 10:07:30 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-30T10:07:31.197 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 10:07:31 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-30T10:07:31.197 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 10:07:31 smithi026 systemd[1]: Stopped Ceph node-exporter.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:07:31.197 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Apr 30 10:07:31 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.a.service: Consumed 3.356s CPU time. 2024-04-30T10:07:31.537 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:31 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:07:31.537 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:31 smithi026 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a[75990]: ts=2024-04-30T10:07:31.495Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-30T10:07:31.805 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:31 smithi026 podman[129261]: 2024-04-30 10:07:31.536581512 +0000 UTC m=+0.145591653 container died 1485dc9c824f0c2df94281f2691f0a787ae72f13650712d92a4f1bf1e68dc02e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T10:07:32.147 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:31 smithi026 podman[129261]: 2024-04-30 10:07:31.895828936 +0000 UTC m=+0.504839202 container cleanup 1485dc9c824f0c2df94281f2691f0a787ae72f13650712d92a4f1bf1e68dc02e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T10:07:32.147 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:31 smithi026 bash[129261]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a 2024-04-30T10:07:32.446 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:32 smithi026 podman[129275]: 2024-04-30 10:07:32.146793834 +0000 UTC m=+0.610589892 container remove 1485dc9c824f0c2df94281f2691f0a787ae72f13650712d92a4f1bf1e68dc02e (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-04-30T10:07:32.446 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:32 smithi026 podman[129275]: 2024-04-30 10:07:32.247413894 +0000 UTC m=+0.711209909 volume remove 449d833dbb33922c451fd2d8781277253c820843224c36a60c6347b058c57921 2024-04-30T10:07:32.918 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:32 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-30T10:07:32.918 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:32 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:07:32.918 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Apr 30 10:07:32 smithi026 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@alertmanager.a.service: Consumed 1.596s CPU time. 2024-04-30T10:07:55.366 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-30T10:07:57.175 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 10:07:56 smithi046 systemd[1]: Stopping Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:07:57.484 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 10:07:57 smithi046 podman[101799]: 2024-04-30 10:07:57.310429093 +0000 UTC m=+0.238050090 container died 0a374c1f04f7dc0e96d76f743bc49d3435c6a3d50f1b02c5dba319af2b828f67 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T10:07:57.945 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 10:07:57 smithi046 podman[101799]: 2024-04-30 10:07:57.699566403 +0000 UTC m=+0.627187434 container cleanup 0a374c1f04f7dc0e96d76f743bc49d3435c6a3d50f1b02c5dba319af2b828f67 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T10:07:57.945 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 10:07:57 smithi046 bash[101799]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b 2024-04-30T10:07:58.234 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 10:07:57 smithi046 podman[101814]: 2024-04-30 10:07:57.944611414 +0000 UTC m=+0.638231796 container remove 0a374c1f04f7dc0e96d76f743bc49d3435c6a3d50f1b02c5dba319af2b828f67 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-04-30T10:07:58.234 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 10:07:57 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-30T10:07:58.234 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 10:07:58 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-30T10:07:58.235 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 10:07:58 smithi046 systemd[1]: Stopped Ceph node-exporter.b for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:07:58.235 INFO:journalctl@ceph.node-exporter.b.smithi046.stdout:Apr 30 10:07:58 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@node-exporter.b.service: Consumed 3.585s CPU time. 2024-04-30T10:07:58.523 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:58 smithi046 systemd[1]: Stopping Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968... 2024-04-30T10:07:58.790 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=server t=2024-04-30T10:07:58.522687508Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-04-30T10:07:58.790 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=ticker t=2024-04-30T10:07:58.523032863Z level=info msg=stopped last_tick=2024-04-30T10:07:50Z 2024-04-30T10:07:58.790 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:58 smithi046 ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a[64086]: logger=tracing t=2024-04-30T10:07:58.523101848Z level=info msg="Closing tracing" 2024-04-30T10:07:58.790 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:58 smithi046 podman[101920]: 2024-04-30 10:07:58.532919191 +0000 UTC m=+0.124896852 container died a88d0112f71cb0e765405764b978160c3055be65b7bbde5c707937d8b8d772d7 (image=quay.io/ceph/grafana:9.4.12, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-30T10:07:59.234 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:58 smithi046 podman[101920]: 2024-04-30 10:07:58.897400496 +0000 UTC m=+0.489378167 container cleanup a88d0112f71cb0e765405764b978160c3055be65b7bbde5c707937d8b8d772d7 (image=quay.io/ceph/grafana:9.4.12, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-30T10:07:59.234 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:58 smithi046 bash[101920]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a 2024-04-30T10:07:59.650 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:59 smithi046 podman[101934]: 2024-04-30 10:07:59.248385854 +0000 UTC m=+0.713573818 container remove a88d0112f71cb0e765405764b978160c3055be65b7bbde5c707937d8b8d772d7 (image=quay.io/ceph/grafana:9.4.12, name=ceph-703f4aea-06d3-11ef-bc93-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-04-30T10:07:59.650 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:59 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@grafana.a.service: Deactivated successfully. 2024-04-30T10:07:59.650 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:59 smithi046 systemd[1]: Stopped Ceph grafana.a for 703f4aea-06d3-11ef-bc93-c7b262605968. 2024-04-30T10:07:59.650 INFO:journalctl@ceph.grafana.a.smithi046.stdout:Apr 30 10:07:59 smithi046 systemd[1]: ceph-703f4aea-06d3-11ef-bc93-c7b262605968@grafana.a.service: Consumed 11.801s CPU time. 2024-04-30T10:08:11.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-30T10:08:11.214 DEBUG:teuthology.orchestra.run.smithi046:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-30T10:08:11.241 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-30T10:08:11.243 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692/remote/smithi026/crash 2024-04-30T10:08:11.244 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/crash -- . 2024-04-30T10:08:11.293 INFO:teuthology.orchestra.run.smithi026.stderr:tar: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-30T10:08:11.293 INFO:teuthology.orchestra.run.smithi026.stderr:tar: Error is not recoverable: exiting now 2024-04-30T10:08:11.295 DEBUG:teuthology.misc:Transferring archived files from smithi046:/var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692/remote/smithi046/crash 2024-04-30T10:08:11.296 DEBUG:teuthology.orchestra.run.smithi046:> sudo tar c -f - -C /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/crash -- . 2024-04-30T10:08:11.329 INFO:teuthology.orchestra.run.smithi046.stderr:tar: /var/lib/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-30T10:08:11.329 INFO:teuthology.orchestra.run.smithi046.stderr:tar: Error is not recoverable: exiting now 2024-04-30T10:08:11.330 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-30T10:08:11.331 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-30T10:08:11.364 INFO:teuthology.orchestra.run.smithi026.stdout:2024-04-30T09:52:29.457352+0000 mon.a (mon.0) 1148 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T10:08:11.365 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-30T10:08:11.365 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[SEC\]' /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-30T10:08:11.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[ERR\]' /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-30T10:08:11.503 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[WRN\]' /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-30T10:08:11.571 INFO:teuthology.orchestra.run.smithi026.stdout:2024-04-30T09:52:29.457352+0000 mon.a (mon.0) 1148 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) ['daemon iscsi.foo.smithi026.romfrr on smithi026 is in unknown state'] (CEPHADM_FAILED_DAEMON) 2024-04-30T10:08:11.571 INFO:tasks.cephadm:Compressing logs... 2024-04-30T10:08:11.571 DEBUG:teuthology.orchestra.run.smithi026:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-30T10:08:11.613 DEBUG:teuthology.orchestra.run.smithi046:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-30T10:08:11.636 INFO:teuthology.orchestra.run.smithi026.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose --: No such file or directory /var/log/ceph/cephadm.log 2024-04-30T10:08:11.636 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T10:08:11.636 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log 2024-04-30T10:08:11.637 INFO:teuthology.orchestra.run.smithi046.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-30T10:08:11.638 INFO:teuthology.orchestra.run.smithi046.stderr:: No such file or directory 2024-04-30T10:08:11.638 INFO:teuthology.orchestra.run.smithi046.stderr:gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log 2024-04-30T10:08:11.638 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-30T10:08:11.639 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.2.log 2024-04-30T10:08:11.639 INFO:teuthology.orchestra.run.smithi026.stderr:gzip/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log: -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-30T10:08:11.639 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.3.log 2024-04-30T10:08:11.639 INFO:teuthology.orchestra.run.smithi026.stderr:gzip/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.audit.log: -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.1.log 2024-04-30T10:08:11.639 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mgr.y.log 2024-04-30T10:08:11.641 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.0.log 2024-04-30T10:08:11.641 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mon.c.log 2024-04-30T10:08:11.642 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.7.log 2024-04-30T10:08:11.643 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.5.log 2024-04-30T10:08:11.643 INFO:teuthology.orchestra.run.smithi046.stderr:gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.4.log 2024-04-30T10:08:11.643 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.7.log: /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mgr.x.log 2024-04-30T10:08:11.643 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.6.log 2024-04-30T10:08:11.643 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-30T10:08:11.643 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.6.log: 89.1% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-30T10:08:11.644 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.0.log: 93.9% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-30T10:08:11.645 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mon.a.log 2024-04-30T10:08:11.647 INFO:teuthology.orchestra.run.smithi046.stderr: 91.1% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-30T10:08:11.647 INFO:teuthology.orchestra.run.smithi046.stderr:gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi046.wmzons.log 2024-04-30T10:08:11.648 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-30T10:08:11.648 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi046.wmzons.log: 76.0% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi046.wmzons.log.gz 2024-04-30T10:08:11.648 INFO:teuthology.orchestra.run.smithi046.stderr:gzip 90.4% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-30T10:08:11.648 INFO:teuthology.orchestra.run.smithi046.stderr: -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mon.b.log 2024-04-30T10:08:11.650 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mon.c.log: 94.5% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-30T10:08:11.654 INFO:teuthology.orchestra.run.smithi026.stderr: 92.3% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-30T10:08:11.658 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-30T10:08:11.658 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/tcmu-runner.log 2024-04-30T10:08:11.661 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.cephadm.log: 94.3% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-30T10:08:11.661 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-30T10:08:11.662 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/tcmu-runner.log: 87.0% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/tcmu-runner.log.gz 2024-04-30T10:08:11.662 INFO:teuthology.orchestra.run.smithi026.stderr:gzip -5 --verbose -- /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi026.igccwv.log 2024-04-30T10:08:11.664 INFO:teuthology.orchestra.run.smithi046.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mon.b.log: 91.0% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-30T10:08:11.674 INFO:teuthology.orchestra.run.smithi046.stderr: 90.7% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-30T10:08:11.676 INFO:teuthology.orchestra.run.smithi026.stderr:/var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi026.igccwv.log: 76.0% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-client.rgw.foo.smithi026.igccwv.log.gz 2024-04-30T10:08:11.680 INFO:teuthology.orchestra.run.smithi026.stderr: 91.0% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-30T10:08:11.803 INFO:teuthology.orchestra.run.smithi046.stderr: 89.8% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mgr.x.log.gz 2024-04-30T10:08:12.176 INFO:teuthology.orchestra.run.smithi026.stderr: 89.2% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mgr.y.log.gz 2024-04-30T10:08:12.476 INFO:teuthology.orchestra.run.smithi026.stderr: 92.4% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mon.c.log.gz 2024-04-30T10:08:12.527 INFO:teuthology.orchestra.run.smithi046.stderr: 92.4% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mon.b.log.gz 2024-04-30T10:08:13.227 INFO:teuthology.orchestra.run.smithi026.stderr: 91.1% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-mon.a.log.gz 2024-04-30T10:08:13.326 INFO:teuthology.orchestra.run.smithi026.stderr: 93.3% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.2.log.gz 2024-04-30T10:08:13.421 INFO:teuthology.orchestra.run.smithi046.stderr: 93.2% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.6.log.gz 2024-04-30T10:08:13.570 INFO:teuthology.orchestra.run.smithi046.stderr: 93.4% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.5.log.gz 2024-04-30T10:08:13.623 INFO:teuthology.orchestra.run.smithi046.stderr: 93.6% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.7.log.gz 2024-04-30T10:08:13.706 INFO:teuthology.orchestra.run.smithi026.stderr: 93.4% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.0.log.gz 2024-04-30T10:08:13.734 INFO:teuthology.orchestra.run.smithi046.stderr: 93.5% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.4.log.gz 2024-04-30T10:08:13.736 INFO:teuthology.orchestra.run.smithi046.stderr: 2024-04-30T10:08:13.736 INFO:teuthology.orchestra.run.smithi046.stderr:real 0m2.108s 2024-04-30T10:08:13.736 INFO:teuthology.orchestra.run.smithi046.stderr:user 0m8.615s 2024-04-30T10:08:13.736 INFO:teuthology.orchestra.run.smithi046.stderr:sys 0m0.295s 2024-04-30T10:08:14.050 INFO:teuthology.orchestra.run.smithi026.stderr: 93.4% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.3.log.gz 2024-04-30T10:08:14.065 INFO:teuthology.orchestra.run.smithi026.stderr: 93.5% -- replaced with /var/log/ceph/703f4aea-06d3-11ef-bc93-c7b262605968/ceph-osd.1.log.gz 2024-04-30T10:08:14.067 INFO:teuthology.orchestra.run.smithi026.stderr: 2024-04-30T10:08:14.067 INFO:teuthology.orchestra.run.smithi026.stderr:real 0m2.441s 2024-04-30T10:08:14.067 INFO:teuthology.orchestra.run.smithi026.stderr:user 0m11.220s 2024-04-30T10:08:14.067 INFO:teuthology.orchestra.run.smithi026.stderr:sys 0m0.373s 2024-04-30T10:08:14.067 INFO:tasks.cephadm:Archiving logs... 2024-04-30T10:08:14.068 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692/remote/smithi026/log 2024-04-30T10:08:14.069 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-30T10:08:15.585 DEBUG:teuthology.misc:Transferring archived files from smithi046:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692/remote/smithi046/log 2024-04-30T10:08:15.586 DEBUG:teuthology.orchestra.run.smithi046:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-30T10:08:16.924 INFO:tasks.cephadm:Removing cluster... 2024-04-30T10:08:16.924 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 --force 2024-04-30T10:08:17.230 DEBUG:teuthology.orchestra.run.smithi046:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 703f4aea-06d3-11ef-bc93-c7b262605968 --force 2024-04-30T10:08:17.534 INFO:tasks.cephadm:Removing cephadm ... 2024-04-30T10:08:17.534 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-30T10:08:17.550 DEBUG:teuthology.orchestra.run.smithi046:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-30T10:08:17.566 INFO:tasks.cephadm:Teardown complete 2024-04-30T10:08:17.566 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-30T10:08:17.581 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-30T10:08:17.581 DEBUG:teuthology.orchestra.run.smithi026:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-30T10:08:17.593 DEBUG:teuthology.orchestra.run.smithi046:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-30T10:08:17.605 INFO:teuthology.orchestra.run.smithi026.stderr:bash: line 1: ntpq: command not found 2024-04-30T10:08:17.610 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-30T10:08:17.610 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2024-04-30T10:08:17.610 INFO:teuthology.orchestra.run.smithi026.stdout:^* hv01.front.sepia.ceph.com 3 7 377 107 +1146us[+1113us] +/- 62ms 2024-04-30T10:08:17.610 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 43 -1210us[-1210us] +/- 62ms 2024-04-30T10:08:17.610 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv03.front.sepia.ceph.com 3 8 377 103 -99us[ -99us] +/- 84ms 2024-04-30T10:08:17.610 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T10:08:17.620 INFO:teuthology.orchestra.run.smithi046.stderr:bash: line 1: ntpq: command not found 2024-04-30T10:08:17.625 INFO:teuthology.orchestra.run.smithi046.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-30T10:08:17.625 INFO:teuthology.orchestra.run.smithi046.stdout:=============================================================================== 2024-04-30T10:08:17.625 INFO:teuthology.orchestra.run.smithi046.stdout:^* hv01.front.sepia.ceph.com 3 7 377 105 +1089us[+1089us] +/- 62ms 2024-04-30T10:08:17.625 INFO:teuthology.orchestra.run.smithi046.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 36 -1267us[-1267us] +/- 63ms 2024-04-30T10:08:17.625 INFO:teuthology.orchestra.run.smithi046.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 38 -206us[ -206us] +/- 85ms 2024-04-30T10:08:17.625 INFO:teuthology.orchestra.run.smithi046.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-04-30T10:08:17.626 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-30T10:08:17.642 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-30T10:08:17.642 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-30T10:08:17.702 DEBUG:teuthology.orchestra.run.smithi026:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-30T10:08:17.858 DEBUG:teuthology.orchestra.run.smithi046:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-30T10:08:17.935 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-30T10:08:17.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-30T10:08:17.964 DEBUG:teuthology.orchestra.run.smithi046:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2024-04-30T10:08:17.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-30T10:08:17.993 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-30T10:08:18.002 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-30T10:08:18.016 INFO:teuthology.task.internal:Duration was 2875.234612 seconds 2024-04-30T10:08:18.016 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-30T10:08:18.047 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-30T10:08:18.048 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-30T10:08:18.050 DEBUG:teuthology.orchestra.run.smithi046:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-30T10:08:18.086 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-30T10:08:18.088 INFO:teuthology.orchestra.run.smithi046.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-30T10:08:18.485 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-30T10:08:18.485 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi026.front.sepia.ceph.com 2024-04-30T10:08:18.485 DEBUG:teuthology.orchestra.run.smithi026:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-30T10:08:18.546 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi046.front.sepia.ceph.com 2024-04-30T10:08:18.546 DEBUG:teuthology.orchestra.run.smithi046:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-30T10:08:18.573 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-30T10:08:18.573 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-30T10:08:18.588 DEBUG:teuthology.orchestra.run.smithi046:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-30T10:08:18.644 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-30T10:08:18.645 DEBUG:teuthology.orchestra.run.smithi026:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-30T10:08:18.655 DEBUG:teuthology.orchestra.run.smithi046:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-30T10:08:19.306 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-30T10:08:19.316 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-30T10:08:19.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-30T10:08:19.341 DEBUG:teuthology.orchestra.run.smithi046:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-30T10:08:19.366 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-30T10:08:19.421 DEBUG:teuthology.orchestra.run.smithi026:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-30T10:08:19.423 DEBUG:teuthology.orchestra.run.smithi046:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-30T10:08:19.443 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = core 2024-04-30T10:08:19.446 INFO:teuthology.orchestra.run.smithi046.stdout:kernel.core_pattern = core 2024-04-30T10:08:19.467 DEBUG:teuthology.orchestra.run.smithi026:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-30T10:08:19.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-30T10:08:19.516 DEBUG:teuthology.orchestra.run.smithi046:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-30T10:08:19.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-30T10:08:19.531 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-30T10:08:19.545 INFO:teuthology.task.internal:Transferring archived files... 2024-04-30T10:08:19.545 DEBUG:teuthology.misc:Transferring archived files from smithi026:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692/remote/smithi026 2024-04-30T10:08:19.546 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-30T10:08:19.609 DEBUG:teuthology.misc:Transferring archived files from smithi046:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692/remote/smithi046 2024-04-30T10:08:19.609 DEBUG:teuthology.orchestra.run.smithi046:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-30T10:08:19.649 INFO:teuthology.task.internal:Removing archive directory... 2024-04-30T10:08:19.649 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-30T10:08:19.651 DEBUG:teuthology.orchestra.run.smithi046:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-30T10:08:19.698 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-30T10:08:19.711 INFO:teuthology.task.internal:Not uploading archives. 2024-04-30T10:08:19.711 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-30T10:08:19.808 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-30T10:08:19.809 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-30T10:08:19.811 DEBUG:teuthology.orchestra.run.smithi046:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-30T10:08:19.824 INFO:teuthology.orchestra.run.smithi026.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 30 10:08 /home/ubuntu/cephtest 2024-04-30T10:08:19.829 INFO:teuthology.orchestra.run.smithi046.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 30 10:08 /home/ubuntu/cephtest 2024-04-30T10:08:19.830 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-30T10:08:19.841 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-30T10:08:19.964 INFO:teuthology.nuke:Checking targets against current locks 2024-04-30T10:08:19.991 DEBUG:teuthology.nuke:shortname: smithi026 2024-04-30T10:08:19.991 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-30T10:08:20.014 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-30 09:13:17.215227', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIvrf5XDJXehVvItiF73UHcGXYUQzZpaCBTVhsGK3OpwvamgLxx66itFNXyFn64ATcCOYczjuz15JDFpp/gRvck='} 2024-04-30T10:08:20.017 DEBUG:teuthology.nuke:shortname: smithi046 2024-04-30T10:08:20.018 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-30T10:08:20.041 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi046.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-30_05:42:49-orch:cephadm-wip-adk-testing-2024-04-29-2009-distro-default-smithi/7680692', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-30 09:13:17.216866', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBD0ZpLm0ZZY5MeL4LntGTEJ8dQjcpYJtJNsqvQqPTl6g+TImH47LGBTkFcYpgXdz1Q1j1P05QqY8izRVkMSuuIM='} 2024-04-30T10:08:20.070 INFO:teuthology.orchestra.console.smithi026:Power off 2024-04-30T10:08:20.070 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-30T10:08:20.101 DEBUG:teuthology.orchestra.console.smithi026:power off output: Chassis Power Control: Down/Off 2024-04-30T10:08:20.101 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-30T10:08:20.116 INFO:teuthology.orchestra.console.smithi046:Power off 2024-04-30T10:08:20.116 DEBUG:teuthology.orchestra.console.smithi046:pexpect command: ipmitool -H smithi046.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-30T10:08:20.126 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on 2024-04-30T10:08:20.143 DEBUG:teuthology.orchestra.console.smithi046:power off output: Chassis Power Control: Down/Off 2024-04-30T10:08:20.144 DEBUG:teuthology.orchestra.console.smithi046:pexpect command: ipmitool -H smithi046.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-30T10:08:20.188 DEBUG:teuthology.orchestra.console.smithi046:check power output: Chassis Power is on 2024-04-30T10:08:24.128 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-30T10:08:24.189 DEBUG:teuthology.orchestra.console.smithi046:pexpect command: ipmitool -H smithi046.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-30T10:08:24.238 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on 2024-04-30T10:08:24.299 DEBUG:teuthology.orchestra.console.smithi046:check power output: Chassis Power is on 2024-04-30T10:08:28.242 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-30T10:08:28.300 DEBUG:teuthology.orchestra.console.smithi046:pexpect command: ipmitool -H smithi046.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-30T10:08:28.355 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is off 2024-04-30T10:08:28.417 DEBUG:teuthology.orchestra.console.smithi046:check power output: Chassis Power is off 2024-04-30T10:08:28.456 INFO:teuthology.orchestra.console.smithi026:Power off completed 2024-04-30T10:08:28.518 INFO:teuthology.orchestra.console.smithi046:Power off completed 2024-04-30T10:08:28.622 INFO:teuthology.run:Summary data: description: orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/classic} duration: 2875.2346115112305 failure_reason: '"2024-04-30T09:52:29.457352+0000 mon.a (mon.0) 1148 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) [''daemon iscsi.foo.smithi026.romfrr on smithi026 is in unknown state''] (CEPHADM_FAILED_DAEMON)" in cluster log' owner: scheduled_adking@teuthology success: false 2024-04-30T10:08:28.622 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-30T10:08:28.696 INFO:teuthology.run:FAIL